`backend/src/scraper/cypresFinder.js`

319 lines
const axios = require('axios');
const cheerio = require('cheerio');
const puppeteer = require('puppeteer');
const robotsParser = require('robots-parser');

class CyPresFinder {
  constructor(options = {}) {
    this.delay = options.delay || 2000;
    this.userAgent = options.userAgent || 'CyPresAwards Bot/1.0';
    this.timeout = options.timeout || 30000;
  }

  // Common cy pres URL patterns
  getCyPresUrlPatterns(baseUrl) {
    const domain = new URL(baseUrl).origin;
    return [
      `${domain}/cy-pres`,
      `${domain}/cy-pres/`,
      `${domain}/cypres`,
      `${domain}/cypres/`,
      `${domain}/cy-pres-awards`,
      `${domain}/legal/cy-pres`,
      `${domain}/awards/cy-pres`,
      `${domain}/support/cy-pres`,
      `${domain}/donate/cy-pres`,
      `${domain}/about/cy-pres`,
    ];
  }

  // Check robots.txt before scraping
  async checkRobotsTxt(url) {
    try {
      const domain = new URL(url).origin;
      const robotsUrl = `${domain}/robots.txt`;
      const response = await axios.get(robotsUrl, { timeout: 5000 });
      const robots = robotsParser(robotsUrl, response.data);
      return robots.isAllowed(url, this.userAgent);
    } catch (error) {
      // If robots.txt doesn't exist, assume allowed
      return true;
    }
  }

  // Try to find cy pres page by checking common patterns
  async findCyPresPage(baseUrl) {
    const patterns = this.getCyPresUrlPatterns(baseUrl);

    for (const url of patterns) {
      try {
        await this.sleep(this.delay);

        const response = await axios.get(url, {
          headers: { 'User-Agent': this.userAgent },
          timeout: this.timeout,
          validateStatus: (status) => status === 200,
        });

        if (response.status === 200) {
          // Check if page actually contains cy pres content
          const $ = cheerio.load(response.data);
          const text = $('body').text().toLowerCase();

          if (this.containsCyPresKeywords(text)) {
            console.log(`Found cy pres page: ${url}`);
            return { url, html: response.data };
          }
        }
      } catch (error) {
        // Page doesn't exist, continue to next pattern
        continue;
      }
    }

    // If direct URLs don't work, try searching the site
    return await this.searchSiteForCyPres(baseUrl);
  }

  // Search entire site for cy pres mentions
  async searchSiteForCyPres(baseUrl) {
    try {
      await this.sleep(this.delay);

      const response = await axios.get(baseUrl, {
        headers: { 'User-Agent': this.userAgent },
        timeout: this.timeout,
      });

      const $ = cheerio.load(response.data);

      // Look for links containing cy pres keywords
      const links = [];
      $('a[href]').each((i, elem) => {
        const href = $(elem).attr('href');
        const text = $(elem).text().toLowerCase();

        if (href && (
          href.includes('cy-pres') ||
          href.includes('cypres') ||
          text.includes('cy pres') ||
          text.includes('cy-pres')
        )) {
          let fullUrl = href;
          if (!href.startsWith('http')) {
            const domain = new URL(baseUrl).origin;
            fullUrl = href.startsWith('/') ? `${domain}${href}` : `${domain}/${href}`;
          }
          links.push(fullUrl);
        }
      });

      // Try each found link
      for (const link of links) {
        try {
          await this.sleep(this.delay);
          const linkResponse = await axios.get(link, {
            headers: { 'User-Agent': this.userAgent },
            timeout: this.timeout,
          });

          const linkHtml = linkResponse.data;
          const $link = cheerio.load(linkHtml);
          const linkText = $link('body').text().toLowerCase();

          if (this.containsCyPresKeywords(linkText)) {
            console.log(`Found cy pres page via search: ${link}`);
            return { url: link, html: linkHtml };
          }
        } catch (error) {
          continue;
        }
      }

      return null;
    } catch (error) {
      console.error(`Error searching site ${baseUrl}:`, error.message);
      return null;
    }
  }

  // Check if text contains cy pres-related keywords
  containsCyPresKeywords(text) {
    const keywords = [
      'cy pres',
      'cy-pres',
      'cypres',
      'class action settlement',
      'unclaimed settlement funds',
      'residual funds',
      'settlement distribution',
      'class action award'
    ];

    let matchCount = 0;
    for (const keyword of keywords) {
      if (text.includes(keyword)) {
        matchCount++;
      }
    }

    // Must have at least 2 keyword matches to be considered a cy pres page
    return matchCount >= 2;
  }

  // Extract organization details from page
  async extractOrgDetails(html, url) {
    const $ = cheerio.load(html);

    // Extract mission statement
    let mission = '';
    const missionSelectors = [
      'meta[name="description"]',
      '.mission',
      '#mission',
      '[class*="mission"]',
      '.about',
      '#about'
    ];

    for (const selector of missionSelectors) {
      const element = $(selector).first();
      if (element.length) {
        mission = selector.startsWith('meta')
          ? element.attr('content')
          : element.text().trim();
        if (mission && mission.length > 50) {
          break;
        }
      }
    }

    // Extract logo
    let logo = '';
    const logoSelectors = [
      'img[alt*="logo" i]',
      'img[class*="logo" i]',
      'img[id*="logo" i]',
      '.logo img',
      '#logo img',
      'header img:first'
    ];

    for (const selector of logoSelectors) {
      const element = $(selector).first();
      if (element.length) {
        logo = element.attr('src');
        if (logo && !logo.startsWith('http')) {
          const domain = new URL(url).origin;
          logo = logo.startsWith('/') ? `${domain}${logo}` : `${domain}/${logo}`;
        }
        if (logo) break;
      }
    }

    // Extract organization name
    let name = '';
    const nameSelectors = [
      'meta[property="og:site_name"]',
      'meta[name="application-name"]',
      'title',
      'h1:first',
      '.site-title',
      '#site-title'
    ];

    for (const selector of nameSelectors) {
      const element = $(selector).first();
      if (element.length) {
        name = selector.startsWith('meta')
          ? element.attr('content')
          : element.text().trim();
        if (name) break;
      }
    }

    return {
      name,
      mission: mission.substring(0, 1000), // Limit length
      logo
    };
  }

  // Extract cy pres statement text
  extractCyPresStatement(html) {
    const $ = cheerio.load(html);

    // Remove script and style elements
    $('script, style, nav, footer, header').remove();

    // Get main content
    const mainSelectors = [
      'main',
      '[role="main"]',
      '.main-content',
      '#main-content',
      'article',
      '.content',
      '#content'
    ];

    let statementText = '';
    for (const selector of mainSelectors) {
      const element = $(selector).first();
      if (element.length) {
        statementText = element.text().trim();
        if (statementText.length > 200) {
          break;
        }
      }
    }

    // If no main content found, get body text
    if (!statementText || statementText.length < 200) {
      statementText = $('body').text().trim();
    }

    // Clean up whitespace
    statementText = statementText.replace(/\s+/g, ' ').trim();

    // Extract page title
    const pageTitle = $('title').text().trim() || $('h1').first().text().trim();

    return {
      statement_text: statementText.substring(0, 5000), // Limit length
      page_title: pageTitle.substring(0, 500),
      html_content: html
    };
  }

  // Sleep helper
  sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  // Use Puppeteer for JavaScript-heavy sites
  async scrapeDynamicSite(url) {
    let browser;
    try {
      browser = await puppeteer.launch({
        headless: 'new',
        args: ['--no-sandbox', '--disable-setuid-sandbox']
      });

      const page = await browser.newPage();
      await page.setUserAgent(this.userAgent);
      await page.goto(url, { waitUntil: 'networkidle2', timeout: this.timeout });

      const html = await page.content();
      await browser.close();

      return html;
    } catch (error) {
      if (browser) await browser.close();
      throw error;
    }
  }
}

module.exports = CyPresFinder;