`backend/src/scraper/scrape500.js`

53 lines
// Scrape 500 major non-profits for cy pres statements
const CyPresScraper = require('./main');
const major500Nonprofits = require('../utils/major500Nonprofits');

async function scrape500() {
  console.log('╔════════════════════════════════════════════════════════════╗');
  console.log('║       CyPresAwards - Major 500 Non-Profits Scraper       ║');
  console.log('╚════════════════════════════════════════════════════════════╝\n');

  const scraper = new CyPresScraper();
  const maxConcurrent = parseInt(process.env.SCRAPER_MAX_CONCURRENT) || 5;

  console.log(`📊 Total Organizations: ${major500Nonprofits.length}`);
  console.log(`⚡ Concurrent Requests: ${maxConcurrent}`);
  console.log(`⏱️  Delay Between Requests: ${process.env.SCRAPER_DELAY_MS || 2000}ms`);
  console.log(`🤖 User Agent: ${process.env.SCRAPER_USER_AGENT || 'CyPresAwards Bot/1.0'}`);

  const estimatedMinutes = Math.ceil((major500Nonprofits.length * 3) / maxConcurrent / 60);
  console.log(`⏰ Estimated Time: ${estimatedMinutes} minutes (${Math.floor(estimatedMinutes / 60)}h ${estimatedMinutes % 60}m)\n`);

  console.log('Starting scrape...\n');
  console.log('═'.repeat(60));

  const startTime = Date.now();

  try {
    await scraper.scrapeList(major500Nonprofits, maxConcurrent);

    const endTime = Date.now();
    const totalSeconds = Math.floor((endTime - startTime) / 1000);
    const hours = Math.floor(totalSeconds / 3600);
    const minutes = Math.floor((totalSeconds % 3600) / 60);
    const seconds = totalSeconds % 60;

    console.log('\n' + '═'.repeat(60));
    console.log('\n✅ SCRAPING COMPLETE!\n');
    console.log(`⏱️  Total Time: ${hours}h ${minutes}m ${seconds}s`);
    console.log('\nCheck http://45.61.58.125:8080 to view results!\n');

    process.exit(0);
  } catch (error) {
    console.error('\n❌ Scraping failed:', error.message);
    process.exit(1);
  }
}

// Run if executed directly
if (require.main === module) {
  scrape500();
}

module.exports = scrape500;