`backend/src/scraper/scrapeLawFirms.js`

52 lines
// Scrape law firms for cy pres statements
const CyPresScraper = require('./main');
const lawFirms = require('../utils/lawFirms');

async function scrapeLawFirms() {
  console.log('\n╔════════════════════════════════════════════════════════════╗');
  console.log('║          CyPresAwards - Law Firms Scraper                 ║');
  console.log('╚════════════════════════════════════════════════════════════╝\n');

  const scraper = new CyPresScraper();
  const maxConcurrent = parseInt(process.env.SCRAPER_MAX_CONCURRENT) || 5;

  console.log(`📊 Total Law Firms: ${lawFirms.length}`);
  console.log(`⚡ Concurrent Requests: ${maxConcurrent}`);
  console.log(`⏱️  Delay Between Requests: ${process.env.SCRAPER_DELAY_MS || 2000}ms\n`);

  const estimatedMinutes = Math.ceil((lawFirms.length * 3) / maxConcurrent / 60);
  console.log(`⏰ Estimated Time: ${estimatedMinutes} minutes (${Math.floor(estimatedMinutes / 60)}h ${estimatedMinutes % 60}m)\n`);

  console.log('Starting law firm scrape...\n');
  console.log('═'.repeat(60));

  const startTime = Date.now();

  try {
    await scraper.scrapeList(lawFirms, maxConcurrent);

    const endTime = Date.now();
    const totalSeconds = Math.floor((endTime - startTime) / 1000);
    const hours = Math.floor(totalSeconds / 3600);
    const minutes = Math.floor((totalSeconds % 3600) / 60);
    const seconds = totalSeconds % 60;

    console.log('\n' + '═'.repeat(60));
    console.log('\n✅ LAW FIRM SCRAPING COMPLETE!\n');
    console.log(`⏱️  Total Time: ${hours}h ${minutes}m ${seconds}s`);
    console.log('\nCheck http://45.61.58.125:8080 to view results!\n');

    process.exit(0);
  } catch (error) {
    console.error('\n❌ Law firm scraping failed:', error.message);
    process.exit(1);
  }
}

// Run if executed directly
if (require.main === module) {
  scrapeLawFirms();
}

module.exports = scrapeLawFirms;