`backend/src/scraper/main.js`

258 lines
const CyPresFinder = require('./cypresFinder');
const Organization = require('../models/Organization');
const { query } = require('../models/database');
const natural = require('natural');
require('dotenv').config();

class CyPresScraper {
  constructor() {
    this.finder = new CyPresFinder({
      delay: parseInt(process.env.SCRAPER_DELAY_MS) || 2000,
      userAgent: process.env.SCRAPER_USER_AGENT
    });
    this.classifier = new natural.BayesClassifier();
    this.loadClassifier();
  }

  // Load or train text classifier for categorization
  loadClassifier() {
    // Training data for automatic categorization
    const trainingData = [
      { text: 'legal aid services lawyer attorney court', categories: ['Legal Aid'] },
      { text: 'education school student learning university', categories: ['Education & Research'] },
      { text: 'health medical healthcare clinic hospital', categories: ['Health Services'] },
      { text: 'environment conservation climate nature wildlife', categories: ['Environmental Conservation'] },
      { text: 'community development neighborhood local residents', categories: ['Community Development'] },
      { text: 'arts culture music theater performance', categories: ['Arts & Culture'] },
      { text: 'children youth kids students teens', categories: ['Youth & Children'] },
      { text: 'veterans military service armed forces', categories: ['Veterans Services'] },
      { text: 'disability accessible wheelchair special needs', categories: ['Disability Rights'] },
      { text: 'seniors elderly aging retirement', categories: ['Senior Services'] },
      { text: 'advocacy policy reform legislation government', categories: ['Advocacy & Policy'] },
      { text: 'technology digital internet innovation software', categories: ['Technology & Innovation'] },
      { text: 'animal pets rescue shelter wildlife', categories: ['Animal Welfare'] },
      { text: 'international global humanitarian relief development', categories: ['International Development'] },
      { text: 'social services welfare assistance support', categories: ['Social Services'] }
    ];

    // Train classifier
    trainingData.forEach(item => {
      item.categories.forEach(category => {
        this.classifier.addDocument(item.text, category);
      });
    });

    this.classifier.train();
  }

  // Categorize organization based on mission and content
  categorizeOrganization(text) {
    const categories = [];
    const classification = this.classifier.getClassifications(text.toLowerCase());

    // Take top 3 categories with confidence > 0.1
    classification
      .filter(c => c.value > 0.1)
      .slice(0, 3)
      .forEach(c => categories.push(c.label));

    return categories;
  }

  // Identify legal topics from text
  identifyLegalTopics(text) {
    const lowerText = text.toLowerCase();
    const topics = [];

    const topicKeywords = {
      'Consumer Protection': ['consumer', 'fraud', 'advertising', 'ftc', 'unfair practices'],
      'Civil Rights': ['civil rights', 'discrimination', 'equality', 'voting', 'ada', 'aclu'],
      'Environmental Law': ['environment', 'pollution', 'climate', 'epa', 'conservation'],
      'Labor & Employment': ['labor', 'employment', 'wage', 'workplace', 'union', 'workers'],
      'Privacy & Data Security': ['privacy', 'data breach', 'security', 'gdpr', 'personal information'],
      'Healthcare': ['healthcare', 'medical', 'health insurance', 'medicare', 'medicaid'],
      'Education': ['education', 'school', 'student rights', 'title ix', 'university'],
      'Housing': ['housing', 'tenant', 'homeless', 'eviction', 'fair housing'],
      'Immigration': ['immigration', 'asylum', 'deportation', 'refugee', 'visa', 'immigrant'],
      'Criminal Justice': ['criminal justice', 'prison', 'conviction', 'sentencing', 'incarceration'],
      'Technology & Internet': ['technology', 'internet', 'digital rights', 'net neutrality', 'tech'],
      'Financial Services': ['financial', 'banking', 'securities', 'predatory lending', 'credit']
    };

    for (const [topic, keywords] of Object.entries(topicKeywords)) {
      let matchCount = 0;
      for (const keyword of keywords) {
        if (lowerText.includes(keyword)) {
          matchCount++;
        }
      }
      if (matchCount >= 2) {
        topics.push(topic);
      }
    }

    return topics;
  }

  // Scrape a single non-profit
  async scrapeNonProfit(websiteUrl) {
    console.log(`\nScraping: ${websiteUrl}`);

    try {
      // Check robots.txt
      const allowed = await this.finder.checkRobotsTxt(websiteUrl);
      if (!allowed) {
        console.log(`Robots.txt disallows scraping: ${websiteUrl}`);
        await this.logScraping(websiteUrl, 'blocked', 'Blocked by robots.txt');
        return null;
      }

      // Find cy pres page
      const cyPresPage = await this.finder.findCyPresPage(websiteUrl);

      if (!cyPresPage) {
        console.log(`No cy pres page found: ${websiteUrl}`);
        await this.logScraping(websiteUrl, 'no_cypres_page', 'No cy pres page found');
        return null;
      }

      // Extract organization details
      const orgDetails = await this.finder.extractOrgDetails(cyPresPage.html, websiteUrl);
      const cyPresStatement = this.finder.extractCyPresStatement(cyPresPage.html);

      // Categorize and identify topics
      const combinedText = `${orgDetails.mission} ${cyPresStatement.statement_text}`;
      const categories = this.categorizeOrganization(combinedText);
      const legalTopics = this.identifyLegalTopics(combinedText);

      // Save to database
      const org = await Organization.create({
        name: orgDetails.name || new URL(websiteUrl).hostname,
        website_url: websiteUrl,
        cypres_page_url: cyPresPage.url,
        mission_statement: orgDetails.mission,
        logo_url: orgDetails.logo,
        location_country: 'USA'
      });

      // Add categories and legal topics
      if (categories.length > 0) {
        await Organization.addCategories(org.id, categories);
      }

      if (legalTopics.length > 0) {
        await Organization.addLegalTopics(org.id, legalTopics);
      }

      // Save cy pres statement
      await Organization.saveCyPresStatement(org.id, cyPresStatement);

      await this.logScraping(websiteUrl, 'success', 'Successfully scraped');

      console.log(`✓ Successfully scraped: ${orgDetails.name}`);
      console.log(`  Categories: ${categories.join(', ')}`);
      console.log(`  Legal Topics: ${legalTopics.join(', ')}`);

      return org;

    } catch (error) {
      console.error(`Error scraping ${websiteUrl}:`, error.message);
      await this.logScraping(websiteUrl, 'failed', error.message);
      return null;
    }
  }

  // Log scraping activity
  async logScraping(url, status, message) {
    const sql = `
      INSERT INTO scraping_logs (url, status, error_message)
      VALUES ($1, $2, $3)
    `;
    await query(sql, [url, status, message]);
  }

  // Scrape multiple non-profits from a list
  async scrapeList(urls, maxConcurrent = 5) {
    console.log(`Starting scraping of ${urls.length} organizations...`);
    console.log(`Max concurrent requests: ${maxConcurrent}\n`);

    const results = [];

    for (let i = 0; i < urls.length; i += maxConcurrent) {
      const batch = urls.slice(i, i + maxConcurrent);
      const batchPromises = batch.map(url => this.scrapeNonProfit(url));
      const batchResults = await Promise.allSettled(batchPromises);

      batchResults.forEach(result => {
        if (result.status === 'fulfilled') {
          results.push(result.value);
        }
      });

      console.log(`\nProgress: ${Math.min(i + maxConcurrent, urls.length)}/${urls.length}`);
    }

    const successful = results.filter(r => r !== null).length;
    console.log(`\n=== Scraping Complete ===`);
    console.log(`Total: ${urls.length}`);
    console.log(`Successful: ${successful}`);
    console.log(`Failed: ${urls.length - successful}`);

    return results;
  }
}

// Main execution
if (require.main === module) {
  const scraper = new CyPresScraper();

  const args = process.argv.slice(2);
  const isTest = args.includes('--test');
  const isLarge = args.includes('--large');
  const isMega = args.includes('--mega');

  // Choose URL list based on arguments
  let nonprofitUrls;
  if (isMega) {
    nonprofitUrls = require('../utils/nonprofitUrls_mega');
  } else if (isLarge) {
    nonprofitUrls = require('../utils/nonprofitUrls_large');
  } else {
    nonprofitUrls = require('../utils/nonprofitUrls');
  }

  if (isTest) {
    console.log('Running in TEST mode with 3 sample URLs...\n');
    const testUrls = nonprofitUrls.slice(0, 3);
    scraper.scrapeList(testUrls, 1)
      .then(() => {
        console.log('\nTest complete. Exiting...');
        process.exit(0);
      })
      .catch(error => {
        console.error('Error:', error);
        process.exit(1);
      });
  } else {
    // Production mode - scrape all URLs
    console.log(`Running in PRODUCTION mode with ${nonprofitUrls.length} URLs...\n`);

    const estimatedHours = Math.ceil((nonprofitUrls.length * 2.5) / 3600); // ~2.5 seconds per URL average
    console.log(`Estimated time: ${estimatedHours} hours\n`);

    const maxConcurrent = parseInt(process.env.SCRAPER_MAX_CONCURRENT) || 5;

    scraper.scrapeList(nonprofitUrls, maxConcurrent)
      .then(() => {
        console.log('\nProduction scraping complete. Exiting...');
        process.exit(0);
      })
      .catch(error => {
        console.error('Error:', error);
        process.exit(1);
      });
  }
}

module.exports = CyPresScraper;