`backend/src/scraper/fetch_all_nonprofits.js`

258 lines
// Fetch ALL US Nonprofits from IRS/ProPublica Database
// This will create a comprehensive list of ALL 501(c)(3) organizations

const axios = require('axios');
const fs = require('fs').promises;
const path = require('path');

class AllNonprofitsFetcher {
  constructor() {
    this.outputFile = path.join(__dirname, '../utils/nonprofitUrls_mega.js');
    this.tempFile = path.join(__dirname, '../utils/nonprofits_temp.json');
    this.allOrgs = [];
  }

  // Fetch from ProPublica Nonprofit Explorer API
  // This database has ALL IRS-registered 501(c)(3) organizations
  async fetchFromProPublica() {
    console.log('🚀 Fetching ALL US Nonprofits from ProPublica/IRS Database');
    console.log('Target: 1.8+ million organizations');
    console.log('This will take several hours...\n');

    let page = 0;
    let totalFetched = 0;
    let hasMore = true;

    // State-by-state fetching for better coverage
    const states = [
      'AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA',
      'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
      'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
      'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
      'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY', 'DC'
    ];

    for (const state of states) {
      console.log(`\n📍 Fetching ${state} nonprofits...`);
      page = 0;
      hasMore = true;

      while (hasMore && page < 1000) { // Max 1000 pages per state
        try {
          const response = await axios.get('https://projects.propublica.org/nonprofits/api/v2/search.json', {
            params: {
              state: state,
              c_code: 3, // 501(c)(3) only
              page: page
            },
            headers: {
              'User-Agent': 'Mozilla/5.0 (nonprofit-cy-pres-research)'
            },
            timeout: 30000
          });

          const orgs = response.data.organizations || [];

          if (orgs.length === 0) {
            hasMore = false;
            break;
          }

          // Extract and save org info
          for (const org of orgs) {
            const website = this.extractWebsite(org);
            if (website) {
              this.allOrgs.push({
                ein: org.ein,
                name: org.name,
                city: org.city,
                state: org.state,
                website: website
              });
            }
          }

          totalFetched += orgs.length;
          console.log(`  Page ${page}: +${orgs.length} orgs (Total: ${totalFetched})`);

          page++;

          // Rate limiting - be respectful
          await this.sleep(500);

          // Save progress every 100 pages
          if (page % 100 === 0) {
            await this.saveProgress();
          }

        } catch (error) {
          if (error.response?.status === 429) {
            console.log('  ⏳ Rate limited, waiting 60 seconds...');
            await this.sleep(60000);
          } else {
            console.error(`  Error on page ${page}:`, error.message);
            await this.sleep(5000);
          }
        }
      }

      console.log(`✓ ${state} complete: ${this.allOrgs.filter(o => o.state === state).length} organizations`);
      await this.saveProgress();
    }

    console.log(`\n✅ COMPLETE: ${this.allOrgs.length} total organizations with websites`);
    return this.allOrgs;
  }

  // Extract website from various possible fields
  extractWebsite(org) {
    // Try different possible website fields
    const possibleUrls = [
      org.website,
      org.url,
      org.website_url,
      org.web_address
    ];

    for (const url of possibleUrls) {
      if (url && typeof url === 'string') {
        // Clean and validate URL
        let cleanUrl = url.trim().toLowerCase();

        if (!cleanUrl.startsWith('http')) {
          cleanUrl = 'https://' + cleanUrl;
        }

        // Basic validation
        if (cleanUrl.includes('.') && cleanUrl.length > 10) {
          return cleanUrl;
        }
      }
    }

    return null;
  }

  // Save progress to temp file
  async saveProgress() {
    try {
      await fs.writeFile(
        this.tempFile,
        JSON.stringify({
          timestamp: new Date().toISOString(),
          count: this.allOrgs.length,
          orgs: this.allOrgs
        }, null, 2)
      );
      console.log(`💾 Progress saved: ${this.allOrgs.length} orgs`);
    } catch (error) {
      console.error('Error saving progress:', error.message);
    }
  }

  // Generate final URLs file
  async generateUrlsFile() {
    console.log('\n📝 Generating final URLs file...');

    // Extract unique URLs
    const urls = [...new Set(this.allOrgs.map(o => o.website))];

    console.log(`Unique URLs: ${urls.length}`);

    // Create JavaScript module
    const content = `// ALL US Nonprofit Organizations (501c3)
// Generated: ${new Date().toISOString()}
// Source: IRS/ProPublica Nonprofit Explorer
// Total: ${urls.length} organizations

module.exports = ${JSON.stringify(urls, null, 2)};
`;

    await fs.writeFile(this.outputFile, content);
    console.log(`✅ Saved to ${this.outputFile}`);
  }

  // Alternative: Fetch from IRS state files
  async fetchFromIRS() {
    console.log('\n🏛️ Fetching from IRS Business Master Files...');

    const states = [
      'AL', 'AK', 'AZ', 'AR', 'CA', 'CO', 'CT', 'DE', 'FL', 'GA',
      'HI', 'ID', 'IL', 'IN', 'IA', 'KS', 'KY', 'LA', 'ME', 'MD',
      'MA', 'MI', 'MN', 'MS', 'MO', 'MT', 'NE', 'NV', 'NH', 'NJ',
      'NM', 'NY', 'NC', 'ND', 'OH', 'OK', 'OR', 'PA', 'RI', 'SC',
      'SD', 'TN', 'TX', 'UT', 'VT', 'VA', 'WA', 'WV', 'WI', 'WY', 'DC'
    ];

    for (const state of states) {
      const url = `https://www.irs.gov/pub/irs-soi/eo_${state.toLowerCase()}.csv`;
      console.log(`\nFetching ${state} from IRS...`);

      try {
        const response = await axios.get(url, {
          timeout: 60000,
          responseType: 'text'
        });

        const lines = response.data.split('\n');
        console.log(`  ✓ ${state}: ${lines.length} organizations in IRS file`);

        // Parse CSV and extract URLs
        // Note: IRS CSV format varies, may need adjustment

        await this.sleep(2000);
      } catch (error) {
        console.error(`  ✗ ${state} failed:`, error.message);
      }
    }
  }

  sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
  }

  // Resume from saved progress
  async resumeFromProgress() {
    try {
      const data = await fs.readFile(this.tempFile, 'utf8');
      const saved = JSON.parse(data);
      this.allOrgs = saved.orgs || [];
      console.log(`📂 Resumed from previous progress: ${this.allOrgs.length} orgs`);
      return true;
    } catch (error) {
      console.log('No previous progress found, starting fresh');
      return false;
    }
  }
}

// Main execution
if (require.main === module) {
  const fetcher = new AllNonprofitsFetcher();

  const args = process.argv.slice(2);
  const resume = args.includes('--resume');
  const useIRS = args.includes('--irs');

  (async () => {
    if (resume) {
      await fetcher.resumeFromProgress();
    }

    if (useIRS) {
      await fetcher.fetchFromIRS();
    } else {
      await fetcher.fetchFromProPublica();
    }

    await fetcher.generateUrlsFile();
    console.log('\n✅ All done!');
    process.exit(0);
  })().catch(error => {
    console.error('Fatal error:', error);
    process.exit(1);
  });
}

module.exports = AllNonprofitsFetcher;