← back to Abrams Report

scrapers/scrape-all.js

19 lines

#!/usr/bin/env node
'use strict';
const { scrapeAllRss } = require('./scrape-rss');
const { scrapeAllHtml } = require('./scrape-html');

async function main() {
  console.log('[scrape-all] Starting full scrape at', new Date().toISOString());
  const rssResults = await scrapeAllRss();
  const htmlResults = await scrapeAllHtml();
  const total = [...rssResults, ...htmlResults].reduce((s, r) => s + r.added, 0);
  console.log(`[scrape-all] Done. Total new headlines: ${total}`);
  process.exit(0);
}

main().catch(err => {
  console.error('[scrape-all] Fatal:', err);
  process.exit(1);
});