`backend/src/scraper/scrapeAdditional.js`

52 lines
// Scrape additional non-profits for cy pres statements
const CyPresScraper = require('./main');
const additionalNonprofits = require('../utils/additionalNonprofits');

async function scrapeAdditionalNonprofits() {
  console.log('\n╔════════════════════════════════════════════════════════════╗');
  console.log('║      CyPresAwards - Additional Nonprofits Scraper         ║');
  console.log('╚════════════════════════════════════════════════════════════╝\n');

  const scraper = new CyPresScraper();
  const maxConcurrent = parseInt(process.env.SCRAPER_MAX_CONCURRENT) || 5;

  console.log(`📊 Additional Nonprofits: ${additionalNonprofits.length}`);
  console.log(`⚡ Concurrent Requests: ${maxConcurrent}`);
  console.log(`⏱️  Delay Between Requests: ${process.env.SCRAPER_DELAY_MS || 2000}ms\n`);

  const estimatedMinutes = Math.ceil((additionalNonprofits.length * 3) / maxConcurrent / 60);
  console.log(`⏰ Estimated Time: ${estimatedMinutes} minutes\n`);

  console.log('Starting additional nonprofit scrape...\n');
  console.log('═'.repeat(60));

  const startTime = Date.now();

  try {
    await scraper.scrapeList(additionalNonprofits, maxConcurrent);

    const endTime = Date.now();
    const totalSeconds = Math.floor((endTime - startTime) / 1000);
    const hours = Math.floor(totalSeconds / 3600);
    const minutes = Math.floor((totalSeconds % 3600) / 60);
    const seconds = totalSeconds % 60;

    console.log('\n' + '═'.repeat(60));
    console.log('\n✅ ADDITIONAL NONPROFIT SCRAPING COMPLETE!\n');
    console.log(`⏱️  Total Time: ${hours}h ${minutes}m ${seconds}s`);
    console.log('\nCheck http://45.61.58.125:3500 to view results!\n');

    process.exit(0);
  } catch (error) {
    console.error('\n❌ Additional nonprofit scraping failed:', error.message);
    process.exit(1);
  }
}

// Run if executed directly
if (require.main === module) {
  scrapeAdditionalNonprofits();
}

module.exports = scrapeAdditionalNonprofits;