`backend/src/scraper/courtListenerScraper.js`

305 lines
// CourtListener Legal Case Scraper
// Searches for cy pres award cases using CourtListener/RECAP API

const axios = require('axios');
const { query } = require('../models/database');
require('dotenv').config();

class CourtListenerScraper {
  constructor(apiToken) {
    this.apiToken = apiToken || process.env.COURTLISTENER_API_TOKEN;
    this.baseUrl = 'https://www.courtlistener.com/api/rest/v3';
    this.delay = 1000; // 1 second between requests to be respectful
  }

  // Search for cy pres cases
  async searchCyPresCases(options = {}) {
    const {
      query: searchQuery = 'cy pres OR "cy-pres" OR "cypres"',
      dateAfter = '2010-01-01',
      orderBy = 'score desc',
      limit = 20,
      offset = 0
    } = options;

    console.log(`\nSearching CourtListener for: ${searchQuery}`);
    console.log(`Date range: ${dateAfter} to present\n`);

    try {
      const response = await axios.get(`${this.baseUrl}/search/`, {
        params: {
          type: 'o', // o = opinions (case law)
          q: searchQuery,
          filed_after: dateAfter,
          order_by: orderBy,
          format: 'json'
        },
        headers: this.apiToken ? {
          'Authorization': `Token ${this.apiToken}`
        } : {},
        timeout: 30000
      });

      const results = response.data.results || [];
      const count = response.data.count || 0;

      console.log(`✓ Found ${count} total cases`);
      console.log(`✓ Retrieved ${results.length} cases in this batch\n`);

      return {
        count,
        results,
        next: response.data.next
      };

    } catch (error) {
      console.error('Error searching CourtListener:', error.message);
      if (error.response) {
        console.error('Response status:', error.response.status);
        console.error('Response data:', JSON.stringify(error.response.data, null, 2));
      }
      return null;
    }
  }

  // Get detailed case information
  async getCaseDetails(caseId) {
    console.log(`Fetching details for case ID: ${caseId}`);

    try {
      const response = await axios.get(`${this.baseUrl}/opinions/${caseId}/`, {
        params: {
          format: 'json'
        },
        headers: this.apiToken ? {
          'Authorization': `Token ${this.apiToken}`
        } : {},
        timeout: 30000
      });

      return response.data;
    } catch (error) {
      console.error(`Error fetching case ${caseId}:`, error.message);
      return null;
    }
  }

  // Extract cy pres information from case text
  extractCyPresInfo(caseText, caseData) {
    const lowerText = caseText.toLowerCase();

    // Look for cy pres award amounts
    const amountPatterns = [
      /\$[\d,]+(?:\.\d{2})?(?:\s*million|\s*m)?/gi,
      /cy[\s-]*pres.*?(?:award|distribution|fund).*?\$[\d,]+/gi
    ];

    const amounts = [];
    amountPatterns.forEach(pattern => {
      const matches = caseText.match(pattern);
      if (matches) {
        amounts.push(...matches);
      }
    });

    // Look for recipient organizations
    const recipients = [];
    const recipientPatterns = [
      /(?:awarded to|distributed to|granted to)\s+([A-Z][A-Za-z\s&,]+(?:Foundation|Organization|Society|Institute|Association|Fund|Center|Council|Alliance))/g,
      /(?:beneficiary|beneficiaries):\s*([A-Z][A-Za-z\s&,]+)/g
    ];

    recipientPatterns.forEach(pattern => {
      let match;
      while ((match = pattern.exec(caseText)) !== null) {
        recipients.push(match[1].trim());
      }
    });

    // Extract settlement/class action info
    const isClassAction = lowerText.includes('class action') || lowerText.includes('class-action');
    const isSettlement = lowerText.includes('settlement') || lowerText.includes('settle');

    return {
      amounts,
      recipients: [...new Set(recipients)], // Remove duplicates
      isClassAction,
      isSettlement,
      containsCyPres: lowerText.includes('cy pres') || lowerText.includes('cy-pres') || lowerText.includes('cypres')
    };
  }

  // Save case to database
  async saveCase(caseData, cyPresInfo) {
    const sql = `
      INSERT INTO legal_cases
      (case_name, case_number, court, filed_date, docket_url, opinion_url,
       judge, cy_pres_amount, recipients, is_class_action, case_summary, full_text)
      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
      ON CONFLICT (case_number, court)
      DO UPDATE SET
        case_name = EXCLUDED.case_name,
        opinion_url = EXCLUDED.opinion_url,
        cy_pres_amount = EXCLUDED.cy_pres_amount,
        recipients = EXCLUDED.recipients,
        updated_at = CURRENT_TIMESTAMP
      RETURNING *
    `;

    const values = [
      caseData.case_name || caseData.caseName,
      caseData.docket_number || caseData.docketNumber || 'UNKNOWN',
      caseData.court || 'UNKNOWN',
      caseData.date_filed || caseData.dateFiled || new Date().toISOString(),
      caseData.docket_url || null,
      caseData.absolute_url || caseData.download_url || null,
      caseData.judge || null,
      cyPresInfo.amounts.length > 0 ? cyPresInfo.amounts.join(', ') : null,
      JSON.stringify(cyPresInfo.recipients),
      cyPresInfo.isClassAction,
      caseData.snippet || null,
      caseData.plain_text || null
    ];

    try {
      const result = await query(sql, values);
      console.log(`✓ Saved case: ${caseData.case_name}`);
      return result.rows[0];
    } catch (error) {
      console.error(`Error saving case ${caseData.case_name}:`, error.message);
      return null;
    }
  }

  // Main scraping function
  async scrapeAllCyPresCases(options = {}) {
    console.log('🚀 Starting CourtListener Cy Pres Case Scraper\n');

    const {
      maxCases = 1000,
      dateAfter = '2010-01-01',
      batchSize = 20
    } = options;

    let totalProcessed = 0;
    let totalSaved = 0;
    let offset = 0;

    while (totalProcessed < maxCases) {
      // Search for cases
      const searchResults = await this.searchCyPresCases({
        dateAfter,
        limit: batchSize,
        offset
      });

      if (!searchResults || searchResults.results.length === 0) {
        console.log('No more results found.');
        break;
      }

      // Process each case
      for (const caseResult of searchResults.results) {
        totalProcessed++;

        console.log(`\nProcessing case ${totalProcessed}/${Math.min(maxCases, searchResults.count)}`);
        console.log(`  Case: ${caseResult.caseName || caseResult.case_name}`);
        console.log(`  Court: ${caseResult.court}`);
        console.log(`  Date: ${caseResult.dateFiled || caseResult.date_filed}`);

        // Extract cy pres info
        const cyPresInfo = this.extractCyPresInfo(
          caseResult.text || caseResult.plain_text || '',
          caseResult
        );

        if (cyPresInfo.containsCyPres) {
          console.log(`  ✓ Contains cy pres language`);
          if (cyPresInfo.recipients.length > 0) {
            console.log(`  ✓ Found ${cyPresInfo.recipients.length} potential recipients`);
          }
          if (cyPresInfo.amounts.length > 0) {
            console.log(`  ✓ Found ${cyPresInfo.amounts.length} monetary amounts`);
          }

          // Save to database
          const saved = await this.saveCase(caseResult, cyPresInfo);
          if (saved) {
            totalSaved++;
          }
        } else {
          console.log(`  ⊘ No cy pres content found (false positive)`);
        }

        // Delay to be respectful
        await new Promise(resolve => setTimeout(resolve, this.delay));

        if (totalProcessed >= maxCases) {
          break;
        }
      }

      offset += batchSize;
      console.log(`\n=== Progress: ${totalProcessed} processed, ${totalSaved} saved ===\n`);
    }

    console.log('\n=== Scraping Complete ===');
    console.log(`Total cases processed: ${totalProcessed}`);
    console.log(`Cases with cy pres content saved: ${totalSaved}`);
    console.log(`Success rate: ${((totalSaved / totalProcessed) * 100).toFixed(1)}%`);

    return {
      totalProcessed,
      totalSaved
    };
  }
}

// Main execution
if (require.main === module) {
  const apiToken = process.env.COURTLISTENER_API_TOKEN;

  if (!apiToken) {
    console.log('⚠️  No COURTLISTENER_API_TOKEN found in .env');
    console.log('   You can still use the API but with rate limits.');
    console.log('   Get a free token at: https://www.courtlistener.com/sign-in/\n');
  }

  const scraper = new CourtListenerScraper(apiToken);

  const args = process.argv.slice(2);
  const isTest = args.includes('--test');

  if (isTest) {
    console.log('Running in TEST mode (20 cases)...\n');
    scraper.scrapeAllCyPresCases({
      maxCases: 20,
      dateAfter: '2020-01-01'
    })
      .then(() => {
        console.log('\nTest complete. Exiting...');
        process.exit(0);
      })
      .catch(error => {
        console.error('Error:', error);
        process.exit(1);
      });
  } else {
    console.log('Running in PRODUCTION mode (1000 cases)...\n');
    scraper.scrapeAllCyPresCases({
      maxCases: 1000,
      dateAfter: '2010-01-01'
    })
      .then(() => {
        console.log('\nProduction scraping complete. Exiting...');
        process.exit(0);
      })
      .catch(error => {
        console.error('Error:', error);
        process.exit(1);
      });
  }
}

module.exports = CourtListenerScraper;