`scrapers/scrape-rss.js`

70 lines
'use strict';
const Parser = require('rss-parser');
const crypto = require('crypto');
const { upsertHeadline, logScrape } = require('./db');
const { RSS_SOURCES } = require('./sources');

const parser = new Parser({
  timeout: 15000,
  headers: {
    'User-Agent': 'AbramsDrReport/1.0 (+https://abramsreport.agentabrams.com)',
  },
});

function makeUid(source, url) {
  return crypto.createHash('sha1').update(source + '|' + url).digest('hex').slice(0, 16);
}

function isBreaking(title) {
  return /breaking|emergency|urgent|just in/i.test(title);
}
function isDeveloping(title) {
  return /developing|exclusive|update:|report:/i.test(title);
}

async function scrapeRss(src) {
  let feed;
  try {
    feed = await parser.parseURL(src.rssUrl);
  } catch (err) {
    console.error(`[rss] ${src.key} failed: ${err.message}`);
    logScrape(src.key, 0, err.message);
    return 0;
  }

  let added = 0;
  for (const item of (feed.items || [])) {
    const title = (item.title || '').trim();
    const url = item.link || item.guid;
    if (!title || !url) continue;
    // Headlines + links only — no excerpt, no images (copyright safe)
    const uid = makeUid(src.key, url);
    const pubDate = item.pubDate ? Math.floor(new Date(item.pubDate).getTime() / 1000) : null;
    added += upsertHeadline({
      uid,
      title,
      url,
      source: src.name,
      category: src.category,
      pub_date: pubDate,
      is_breaking: isBreaking(title) ? 1 : 0,
      is_developing: isDeveloping(title) ? 1 : 0,
      is_top: 0,
    });
  }
  logScrape(src.key, added);
  return added;
}

async function scrapeAllRss() {
  const results = [];
  for (const src of RSS_SOURCES) {
    const count = await scrapeRss(src);
    console.log(`[rss] ${src.name}: +${count}`);
    results.push({ source: src.key, added: count });
  }
  return results;
}

module.exports = { scrapeAllRss, scrapeRss };