`cypressaward/scraper/master-awards-list.js`

114 lines
const crypto = require('crypto');

// Import all award files
const { comprehensiveCyPresAwards } = require('./comprehensive-awards-crawler');
const { cypresAwardsData } = require('./cypres-awards-crawler');
const { expandedCyPresAwards } = require('./expanded-awards-crawler');
const { twentyMoreCyPresAwards } = require('./twenty-more-cypres-awards');

// Combine all awards
const allAwards = [
  ...comprehensiveCyPresAwards,
  ...cypresAwardsData,
  ...expandedCyPresAwards,
  ...twentyMoreCyPresAwards
];

// Remove duplicates based on hash
const uniqueAwards = [];
const seenHashes = new Set();

allAwards.forEach(award => {
  if (!award.hash) {
    const uniqueString = `${award.recipient_organization}-${award.case_name}-${award.amount}`;
    award.hash = crypto.createHash('md5').update(uniqueString).digest('hex');
  }
  
  if (!seenHashes.has(award.hash)) {
    seenHashes.add(award.hash);
    uniqueAwards.push(award);
  }
});

// Sort by date (newest first) then by organization name
uniqueAwards.sort((a, b) => {
  const dateA = new Date(a.date_awarded);
  const dateB = new Date(b.date_awarded);
  if (dateB.getTime() !== dateA.getTime()) {
    return dateB.getTime() - dateA.getTime();
  }
  return a.recipient_organization.localeCompare(b.recipient_organization);
});

console.log('═'.repeat(80));
console.log('🏆 MASTER CY PRES AWARDS DATABASE');
console.log('═'.repeat(80));

console.log(`\n📊 Summary Statistics:`);
console.log(`   Total Awards (before deduplication): ${allAwards.length}`);
console.log(`   Unique Awards: ${uniqueAwards.length}`);
console.log(`   Duplicates Removed: ${allAwards.length - uniqueAwards.length}`);

// Calculate total amount
const totalAmount = uniqueAwards.reduce((sum, award) => sum + award.amount, 0);
console.log(`   Total Amount Distributed: $${totalAmount.toLocaleString()}`);

// Count by year
const yearCounts = {};
uniqueAwards.forEach(award => {
  const year = award.year || new Date(award.date_awarded).getFullYear();
  yearCounts[year] = (yearCounts[year] || 0) + 1;
});

console.log(`\n📅 Awards by Year:`);
Object.entries(yearCounts)
  .sort((a, b) => b[0] - a[0])
  .forEach(([year, count]) => {
    console.log(`   ${year}: ${count} awards`);
  });

// Top recipients by amount
console.log(`\n💰 Top 10 Recipients by Amount:`);
uniqueAwards
  .sort((a, b) => b.amount - a.amount)
  .slice(0, 10)
  .forEach((award, index) => {
    console.log(`   ${index + 1}. ${award.recipient_organization}: ${award.amount_text}`);
    console.log(`      Case: ${award.case_name}`);
  });

// Count by court
const courtCounts = {};
uniqueAwards.forEach(award => {
  if (award.court) {
    courtCounts[award.court] = (courtCounts[award.court] || 0) + 1;
  }
});

console.log(`\n⚖️ Top Courts by Number of Awards:`);
Object.entries(courtCounts)
  .sort((a, b) => b[1] - a[1])
  .slice(0, 5)
  .forEach(([court, count]) => {
    console.log(`   ${court}: ${count} awards`);
  });

// Export for use in other scripts
module.exports = { 
  allCyPresAwards: uniqueAwards,
  totalCount: uniqueAwards.length,
  totalAmount: totalAmount
};

// Display all awards
console.log(`\n📋 Complete List of ${uniqueAwards.length} Unique Awards:`);
console.log('─'.repeat(80));

uniqueAwards.forEach((award, index) => {
  console.log(`\n${index + 1}. ${award.recipient_organization}`);
  console.log(`   Amount: ${award.amount_text}`);
  console.log(`   Case: ${award.case_name}`);
  console.log(`   Court: ${award.court}`);
  console.log(`   Date: ${award.date_awarded}`);
  console.log(`   Purpose: ${award.purpose.substring(0, 100)}${award.purpose.length > 100 ? '...' : ''}`);
});