← back to Sdcc Awards

cypressaward/crawler/src/crawlers/court-crawler.ts

174 lines

import { Page } from 'playwright';
import { BaseCrawler, CrawlConfig } from './base-crawler';
import { Pool } from 'pg';
import Redis from 'ioredis';
import { z } from 'zod';

const CaseSchema = z.object({
  title: z.string(),
  docket: z.string().optional(),
  court: z.string(),
  filingDate: z.string().optional(),
  parties: z.array(z.string()).optional(),
  summary: z.string().optional(),
  status: z.enum(['pending', 'active', 'settled', 'awarded', 'closed']).optional(),
  amount: z.number().optional(),
  categories: z.array(z.string()).optional(),
});

export class CourtCrawler extends BaseCrawler {
  constructor(db: Pool, redis: Redis) {
    const config: CrawlConfig = {
      name: 'court-crawler',
      startUrls: [
        'https://www.classaction.org/settlements',
        'https://topclassactions.com/category/lawsuit-settlements/open-lawsuit-settlements/',
        'https://www.bigclassaction.com/settlement/',
      ],
      selectors: {
        title: 'h1, h2.case-title, .settlement-title',
        court: '.court-name, .jurisdiction',
        docket: '.docket-number, .case-number',
        date: '.filing-date, .settlement-date, time',
        amount: '.settlement-amount, .award-amount',
        content: '.case-summary, .settlement-details, article',
        links: 'a[href*="settlement"], a[href*="case"]',
      },
      rateLimit: 2,
      maxConcurrency: 2,
    };
    super(config, db, redis);
  }

  protected async extractContent(page: Page): Promise<any> {
    return await page.evaluate((selectors) => {
      const getText = (selector: string) => {
        const el = document.querySelector(selector);
        return el ? el.textContent?.trim() : null;
      };

      const getTexts = (selector: string) => {
        const els = document.querySelectorAll(selector);
        return Array.from(els).map(el => el.textContent?.trim()).filter(Boolean);
      };

      return {
        title: getText(selectors.title),
        court: getText(selectors.court),
        docket: getText(selectors.docket),
        date: getText(selectors.date),
        amount: getText(selectors.amount),
        content: getText(selectors.content),
        allText: document.body.innerText,
      };
    }, this.config.selectors);
  }

  protected async parseContent(page: Page, content: any): Promise<any> {
    const url = page.url();
    
    let categories: string[] = [];
    let status = 'pending';
    
    const lowerContent = (content.allText || '').toLowerCase();
    
    if (lowerContent.includes('education')) categories.push('education');
    if (lowerContent.includes('student') || lowerContent.includes('tuition')) categories.push('education');
    if (lowerContent.includes('consumer')) categories.push('consumer');
    if (lowerContent.includes('privacy') || lowerContent.includes('data breach')) categories.push('privacy');
    if (lowerContent.includes('employment') || lowerContent.includes('wage')) categories.push('labor');
    if (lowerContent.includes('healthcare') || lowerContent.includes('medical')) categories.push('healthcare');
    if (lowerContent.includes('antitrust')) categories.push('antitrust');
    if (lowerContent.includes('securities')) categories.push('securities');
    
    if (lowerContent.includes('cy pres') || lowerContent.includes('cypres')) {
      categories.push('cy-pres-eligible');
    }
    
    if (lowerContent.includes('settled')) status = 'settled';
    if (lowerContent.includes('closed') || lowerContent.includes('expired')) status = 'closed';
    if (lowerContent.includes('awarded')) status = 'awarded';
    
    let amount: number | undefined;
    const amountMatch = content.amount?.match(/\$?([\d,]+(?:\.\d{2})?)/);
    if (amountMatch) {
      amount = parseFloat(amountMatch[1].replace(/,/g, ''));
    }
    
    return {
      title: content.title || 'Untitled Case',
      docket: content.docket,
      court: content.court || this.extractCourtFromUrl(url),
      filingDate: content.date,
      summary: content.content?.substring(0, 1000),
      status,
      amount,
      categories,
      url,
    };
  }

  private extractCourtFromUrl(url: string): string {
    if (url.includes('sdny')) return 'S.D.N.Y.';
    if (url.includes('ndcal')) return 'N.D. Cal.';
    if (url.includes('edtx')) return 'E.D. Tex.';
    return 'Unknown Court';
  }

  protected async processData(data: any, url: string, hash: string): Promise<void> {
    try {
      const validated = CaseSchema.parse(data);
      
      const existingCase = await this.db.query(
        'SELECT id FROM cases WHERE source_hash = $1 OR (title = $2 AND court = $3)',
        [hash, validated.title, validated.court]
      );

      if (existingCase.rows.length > 0) {
        await this.db.query(
          `UPDATE cases 
           SET last_seen_at = NOW(), 
               status = COALESCE($1, status),
               category = $2,
               summary = COALESCE($3, summary)
           WHERE id = $4`,
          [validated.status, validated.categories, validated.summary, existingCase.rows[0].id]
        );
        this.logger.info(`Updated case: ${validated.title}`);
      } else {
        await this.db.query(
          `INSERT INTO cases 
           (title, docket, court, status, category, summary, url, source_hash, filing_date)
           VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
          [
            validated.title,
            validated.docket,
            validated.court,
            validated.status || 'pending',
            validated.categories || [],
            validated.summary,
            url,
            hash,
            validated.filingDate ? new Date(validated.filingDate) : null,
          ]
        );
        this.logger.info(`New case added: ${validated.title}`);
        
        if (validated.amount && validated.amount > 50000) {
          await this.notifyHighValueCase(validated);
        }
      }
    } catch (error) {
      this.logger.error(`Error processing case data:`, error);
    }
  }

  private async notifyHighValueCase(caseData: any): Promise<void> {
    await this.redis.publish('high-value-cases', JSON.stringify({
      title: caseData.title,
      amount: caseData.amount,
      court: caseData.court,
      categories: caseData.categories,
    }));
  }
}