← back to Sdcc Awards

cypressaward/crawler/src/crawlers/base-crawler.ts

188 lines

import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
import { Page } from 'playwright';
import robotsParser from 'robots-txt-parser';
import { createHash } from 'crypto';
import { Pool } from 'pg';
import Redis from 'ioredis';
import winston from 'winston';

export interface CrawlConfig {
  name: string;
  startUrls: string[];
  selectors: {
    title?: string;
    content?: string;
    date?: string;
    amount?: string;
    court?: string;
    docket?: string;
    parties?: string;
    links?: string;
  };
  rateLimit?: number;
  maxConcurrency?: number;
  respectRobotsTxt?: boolean;
}

export abstract class BaseCrawler {
  protected crawler: PlaywrightCrawler;
  protected db: Pool;
  protected redis: Redis;
  protected logger: winston.Logger;
  protected robotsCache: Map<string, any> = new Map();

  constructor(
    protected config: CrawlConfig,
    db: Pool,
    redis: Redis,
  ) {
    this.db = db;
    this.redis = redis;
    
    this.logger = winston.createLogger({
      level: 'info',
      format: winston.format.json(),
      transports: [
        new winston.transports.File({ filename: `logs/${config.name}.log` }),
        new winston.transports.Console({
          format: winston.format.simple(),
        }),
      ],
    });

    this.crawler = new PlaywrightCrawler({
      maxRequestsPerCrawl: 1000,
      maxConcurrency: config.maxConcurrency || 2,
      requestHandlerTimeoutSecs: 30,
      navigationTimeoutSecs: 30,
      
      launchContext: {
        launchOptions: {
          headless: true,
          args: ['--no-sandbox', '--disable-setuid-sandbox'],
        },
      },

      requestHandler: async ({ request, page, enqueueLinks }) => {
        try {
          if (config.respectRobotsTxt !== false) {
            const allowed = await this.checkRobotsTxt(request.url);
            if (!allowed) {
              this.logger.warn(`Robots.txt blocks: ${request.url}`);
              return;
            }
          }

          await this.rateLimitDelay();
          
          const content = await this.extractContent(page);
          const hash = this.generateHash(content);
          
          const isDuplicate = await this.checkDuplicate(hash);
          if (isDuplicate) {
            this.logger.info(`Duplicate content skipped: ${request.url}`);
            return;
          }

          await this.saveCrawlLog(request.url, hash, 200);
          
          const parsed = await this.parseContent(page, content);
          await this.processData(parsed, request.url, hash);

          if (this.config.selectors.links) {
            await enqueueLinks({
              selector: this.config.selectors.links,
              strategy: 'same-domain',
            });
          }
        } catch (error) {
          this.logger.error(`Error crawling ${request.url}:`, error);
          await this.saveCrawlLog(request.url, null, 500, error.message);
        }
      },

      failedRequestHandler: async ({ request, error }) => {
        this.logger.error(`Request ${request.url} failed:`, error);
        await this.saveCrawlLog(request.url, null, 500, error.message);
      },
    });
  }

  protected async checkRobotsTxt(url: string): Promise<boolean> {
    try {
      const urlObj = new URL(url);
      const robotsUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
      
      if (!this.robotsCache.has(robotsUrl)) {
        const response = await fetch(robotsUrl);
        if (response.ok) {
          const text = await response.text();
          const robots = robotsParser(robotsUrl, text);
          this.robotsCache.set(robotsUrl, robots);
        }
      }

      const robots = this.robotsCache.get(robotsUrl);
      if (robots) {
        return robots.isAllowed(url, 'CyPresBot');
      }
    } catch (error) {
      this.logger.warn(`Error checking robots.txt for ${url}:`, error);
    }
    return true;
  }

  protected async rateLimitDelay(): Promise<void> {
    if (this.config.rateLimit) {
      const delay = 1000 / this.config.rateLimit;
      await new Promise(resolve => setTimeout(resolve, delay));
    }
  }

  protected generateHash(content: any): string {
    const str = JSON.stringify(content);
    return createHash('sha256').update(str).digest('hex');
  }

  protected async checkDuplicate(hash: string): Promise<boolean> {
    const cached = await this.redis.get(`content:${hash}`);
    if (cached) return true;

    const result = await this.db.query(
      'SELECT 1 FROM crawl_log WHERE checksum = $1 LIMIT 1',
      [hash]
    );
    
    if (result.rows.length > 0) {
      await this.redis.setex(`content:${hash}`, 900, '1');
      return true;
    }
    
    return false;
  }

  protected async saveCrawlLog(
    url: string,
    checksum: string | null,
    status: number,
    error?: string
  ): Promise<void> {
    const urlObj = new URL(url);
    await this.db.query(
      `INSERT INTO crawl_log 
       (url, domain, http_status, checksum, parser_name, error_message, fetched_at)
       VALUES ($1, $2, $3, $4, $5, $6, NOW())`,
      [url, urlObj.hostname, status, checksum, this.config.name, error]
    );
  }

  protected abstract extractContent(page: Page): Promise<any>;
  protected abstract parseContent(page: Page, content: any): Promise<any>;
  protected abstract processData(data: any, url: string, hash: string): Promise<void>;

  public async run(): Promise<void> {
    this.logger.info(`Starting crawler: ${this.config.name}`);
    await this.crawler.run(this.config.startUrls);
    this.logger.info(`Crawler finished: ${this.config.name}`);
  }
}