`src/crawl/news_scraper.ts`

257 lines
/**
 * News scraper — Phase iii.
 *
 * For every business with a website, fetch the homepage, identify candidate
 * news/blog/press links via common selectors, then fetch up to MAX_PER_SITE
 * articles and extract title + body. Hash + insert into news_items.
 *
 * No Playwright — fetch + cheerio. Stays fast on the long tail (153 sites
 * resolves in ~3min at 4 concurrent / 500ms politeness window).
 *
 * Polite by default:
 *   - One in-flight request per host at a time (per-host queue keyed by hostname)
 *   - 500ms inter-request gap per host
 *   - 8s navigation timeout
 *   - Honors robots.txt Disallow / for User-agent: *
 *   - Custom UA identifying the project + a contact mailto
 *
 * Run:
 *   npm run crawl:news                   # entire corpus, default concurrency 4
 *   tsx src/crawl/news_scraper.ts -- --limit=10  # smoke test
 *   tsx src/crawl/news_scraper.ts -- --business=12345  # one biz only
 */
import 'dotenv/config';
import { createHash } from 'node:crypto';
import * as cheerio from 'cheerio';
import { pool, query } from '../../db/pool.ts';

const UA = 'ventura-corridor/0.1 (news-scraper; +https://leads.venturaclaw.com; contact: steve@designerwallcoverings.com)';
const TIMEOUT_MS = parseInt(process.env.NEWS_TIMEOUT_MS || '8000', 10);
const CONCURRENCY = parseInt(process.env.NEWS_CONCURRENCY || '4', 10);
const HOST_GAP_MS = parseInt(process.env.NEWS_HOST_GAP_MS || '500', 10);
const MAX_PER_SITE = parseInt(process.env.NEWS_MAX_PER_SITE || '3', 10);

const NEWS_KEYWORDS = ['news', 'blog', 'press', 'announcements', 'updates', 'articles', 'posts', 'newsroom', 'media', 'stories', 'journal'];
const BAD_PATH_HINTS = ['login', 'signin', 'signup', 'register', 'cart', 'checkout', 'account', 'wp-admin', 'wp-login', 'feed.xml', '/feed/', '/rss', '.xml', '.json', '/api/', '/static/', '/assets/'];

interface BusinessRow { id: number; name: string; website: string }

const argv = process.argv.slice(2).filter(a => a !== '--');
const limitArg     = parseInt(argv.find(a => a.startsWith('--limit='))?.slice(8)    || '0', 10) || 0;
const businessArg  = parseInt(argv.find(a => a.startsWith('--business='))?.slice(11) || '0', 10) || 0;
const dryRun       = argv.includes('--dry-run');

function sleep(ms: number) { return new Promise<void>(r => setTimeout(r, ms)); }

async function fetchText(url: string): Promise<{ ok: boolean; status: number; text?: string; finalUrl?: string }> {
  try {
    const res = await fetch(url, {
      redirect: 'follow',
      headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml' },
      signal: AbortSignal.timeout(TIMEOUT_MS)
    });
    if (!res.ok) return { ok: false, status: res.status };
    const ctype = res.headers.get('content-type') || '';
    if (!ctype.includes('html')) return { ok: false, status: 415 };
    const text = await res.text();
    return { ok: true, status: res.status, text, finalUrl: res.url };
  } catch (e: any) {
    return { ok: false, status: 0 };
  }
}

const robotsCache = new Map<string, boolean>(); // hostname -> isAllowed
async function isAllowedByRobots(homepageUrl: string): Promise<boolean> {
  let host: string;
  try { host = new URL(homepageUrl).hostname; } catch { return false; }
  if (robotsCache.has(host)) return robotsCache.get(host)!;
  const robotsUrl = new URL('/robots.txt', homepageUrl).toString();
  const r = await fetchText(robotsUrl);
  let allowed = true;
  if (r.ok && r.text) {
    // Crude: if we see "User-agent: *" then "Disallow: /" with nothing more
    // permissive, treat as blocked. Anything else falls through to allowed.
    const lines = r.text.split(/\r?\n/).map(l => l.trim());
    let inStar = false;
    for (const l of lines) {
      const m = l.match(/^([Uu]ser-agent|[Dd]isallow|[Aa]llow):\s*(.*)$/);
      if (!m) continue;
      const k = m[1].toLowerCase(); const v = m[2].trim();
      if (k === 'user-agent') inStar = (v === '*');
      else if (inStar && k === 'disallow' && v === '/') allowed = false;
      else if (inStar && k === 'allow' && v === '/') allowed = true;
    }
  }
  robotsCache.set(host, allowed);
  return allowed;
}

function absUrl(base: string, href: string): string | null {
  try { return new URL(href, base).toString(); } catch { return null; }
}

function looksLikeNewsLink(href: string, text: string): boolean {
  const h = href.toLowerCase(); const t = text.toLowerCase();
  if (BAD_PATH_HINTS.some(b => h.includes(b))) return false;
  return NEWS_KEYWORDS.some(kw => h.includes('/' + kw) || t.includes(kw));
}

function extractCandidateUrls(homepageHtml: string, homepageUrl: string): string[] {
  const $ = cheerio.load(homepageHtml);
  const homeHost = (() => { try { return new URL(homepageUrl).hostname; } catch { return ''; } })();
  const seen = new Set<string>();
  const out: string[] = [];
  $('a[href]').each((_, el) => {
    const href = String($(el).attr('href') || '');
    const text = $(el).text().trim();
    if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;
    if (!looksLikeNewsLink(href, text)) return;
    const abs = absUrl(homepageUrl, href);
    if (!abs) return;
    let host = '';
    try { host = new URL(abs).hostname; } catch { return; }
    if (host !== homeHost) return; // same-origin only
    if (seen.has(abs)) return;
    seen.add(abs);
    out.push(abs);
  });
  return out.slice(0, 12);
}

function extractArticle(html: string, url: string): { title: string; body: string; published?: string } {
  const $ = cheerio.load(html);
  // Title: <article h1> > <main h1> > og:title > <title>
  const title =
       $('article h1').first().text().trim()
    || $('main h1').first().text().trim()
    || $('meta[property="og:title"]').attr('content')?.trim()
    || $('h1').first().text().trim()
    || $('title').text().trim()
    || '';

  // Body: <article> > <main> > biggest .post / .entry / .content > body
  // Strip nav/aside/footer/script/style first.
  ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript', 'form', '.menu', '.nav', '.navigation', '.sidebar'].forEach(sel => $(sel).remove());

  let bodyHtml = $('article').first().html()
              || $('main').first().html()
              || $('.entry-content, .post-content, .blog-content, .article-content').first().html()
              || $('body').html()
              || '';
  const body = cheerio.load(bodyHtml || '').text().replace(/\s+/g, ' ').trim().slice(0, 8000);

  const published = $('meta[property="article:published_time"]').attr('content')
                 || $('meta[name="date"]').attr('content')
                 || $('time[datetime]').first().attr('datetime')
                 || $('time').first().text().trim()
                 || undefined;

  return { title, body, published };
}

async function pickQueue(): Promise<BusinessRow[]> {
  if (businessArg) {
    const r = await query<BusinessRow>(`SELECT id, name, website FROM businesses WHERE id = $1`, [businessArg]);
    return r.rows;
  }
  const r = await query<BusinessRow>(`
    SELECT b.id, b.name, b.website
      FROM businesses b
     WHERE b.website IS NOT NULL
       AND b.website ~ '^https?://'
     ORDER BY
       (SELECT COUNT(*) FROM news_items n WHERE n.business_id = b.id) ASC,  -- under-covered first
       b.id ASC
     ${limitArg ? `LIMIT ${limitArg}` : ''}
  `);
  return r.rows;
}

interface ScrapeResult { businessId: number; inserted: number; skipped: number; error?: string }

const lastFetchByHost = new Map<string, number>();
async function politeFetch(url: string): Promise<{ ok: boolean; text?: string; finalUrl?: string; status: number }> {
  let host = '';
  try { host = new URL(url).hostname; } catch { return { ok: false, status: 0 }; }
  const last = lastFetchByHost.get(host) || 0;
  const wait = (last + HOST_GAP_MS) - Date.now();
  if (wait > 0) await sleep(wait);
  const r = await fetchText(url);
  lastFetchByHost.set(host, Date.now());
  return r;
}

async function scrapeBusiness(b: BusinessRow): Promise<ScrapeResult> {
  const out: ScrapeResult = { businessId: b.id, inserted: 0, skipped: 0 };
  const allowed = await isAllowedByRobots(b.website).catch(() => true);
  if (!allowed) { out.error = 'robots_disallow'; return out; }

  const home = await politeFetch(b.website);
  if (!home.ok || !home.text) { out.error = `home_${home.status}`; return out; }

  const candidates = extractCandidateUrls(home.text, home.finalUrl || b.website);
  if (candidates.length === 0) { out.error = 'no_candidates'; return out; }

  const articleUrls = candidates.slice(0, MAX_PER_SITE);
  for (const url of articleUrls) {
    const r = await politeFetch(url);
    if (!r.ok || !r.text) { out.skipped++; continue; }
    const a = extractArticle(r.text, url);
    if (!a.title || a.body.length < 120) { out.skipped++; continue; }

    const hashSrc = (a.title + '|' + a.body.slice(0, 1000));
    const contentHash = createHash('sha1').update(hashSrc).digest('hex').slice(0, 16);
    const excerpt = a.body.slice(0, 280);
    let publishedGuess: Date | null = null;
    if (a.published) {
      const t = new Date(a.published);
      if (!isNaN(t.getTime())) publishedGuess = t;
    }

    if (dryRun) {
      out.inserted++;
      continue;
    }
    const ins = await query(`
      INSERT INTO news_items (business_id, source_url, title, body, excerpt, published_guess, content_hash)
      VALUES ($1, $2, $3, $4, $5, $6, $7)
      ON CONFLICT (business_id, content_hash) DO NOTHING
    `, [b.id, url, a.title.slice(0, 300), a.body, excerpt, publishedGuess, contentHash]);
    if ((ins.rowCount ?? 0) > 0) out.inserted++; else out.skipped++;
  }
  return out;
}

async function main() {
  const queue = await pickQueue();
  console.log(`[news_scraper] queue=${queue.length} concurrency=${CONCURRENCY} timeout=${TIMEOUT_MS}ms host_gap=${HOST_GAP_MS}ms${dryRun ? ' DRY-RUN' : ''}`);
  let i = 0;
  let totals = { inserted: 0, skipped: 0, errored: 0, businesses_with_finds: 0 };

  // Simple host-pinned scheduler: spawn CONCURRENCY workers; each pulls
  // next item from the queue. Polite-fetch already serializes per-host.
  async function worker() {
    while (true) {
      const idx = i++; if (idx >= queue.length) return;
      const b = queue[idx];
      try {
        const r = await scrapeBusiness(b);
        totals.inserted += r.inserted;
        totals.skipped  += r.skipped;
        if (r.error) totals.errored++;
        if (r.inserted > 0) totals.businesses_with_finds++;
        const tag = r.error ? `· ${r.error}` : `· +${r.inserted} new${r.skipped ? ` / ${r.skipped} dup` : ''}`;
        console.log(`  [${String(idx + 1).padStart(4)}/${queue.length}] ${b.name.padEnd(40)} ${tag}`);
      } catch (e: any) {
        totals.errored++;
        console.warn(`  [${String(idx + 1).padStart(4)}/${queue.length}] ${b.name} · err ${e?.message || e}`);
      }
    }
  }
  await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
  console.log(`[news_scraper] done · inserted=${totals.inserted} skipped=${totals.skipped} errored=${totals.errored} businesses_with_finds=${totals.businesses_with_finds}`);
  await pool.end();
}

main().catch(e => { console.error('FATAL', e); pool.end(); process.exit(1); });