← back to Sdcc Awards

cypressaward/scraper/supabase-scraper.js

332 lines

require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const axios = require('axios');
const cheerio = require('cheerio');
const cron = require('node-cron');
const crypto = require('crypto');

// Supabase configuration
const supabaseUrl = process.env.SUPABASE_URL;
const supabaseKey = process.env.SUPABASE_ANON_KEY;
const supabase = createClient(supabaseUrl, supabaseKey);

// Generate unique hash for deduplication
function generateHash(data) {
  return crypto.createHash('md5').update(JSON.stringify(data)).digest('hex');
}

// Scraping functions for different data types
const scrapers = {
  // Scrape legal cases
  async scrapeCases() {
    const sources = [
      { url: 'https://www.courtlistener.com/api/rest/v3/search/?type=o&q=recent', type: 'api' },
      { url: 'https://www.justia.com/cases/federal/appellate-courts/', type: 'html' }
    ];
    
    const cases = [];
    
    for (const source of sources) {
      try {
        if (source.type === 'api') {
          const response = await axios.get(source.url);
          const data = response.data.results || [];
          
          for (const item of data.slice(0, 20)) {
            const caseData = {
              title: item.caseName || item.name,
              court: item.court || 'Federal Court',
              date_filed: item.dateFiled || new Date().toISOString(),
              docket_number: item.docketNumber || '',
              summary: item.snippet || '',
              url: item.absolute_url || '',
              source: 'CourtListener',
              data_type: 'case',
              hash: ''
            };
            caseData.hash = generateHash(caseData);
            cases.push(caseData);
          }
        } else {
          const response = await axios.get(source.url);
          const $ = cheerio.load(response.data);
          
          $('.case-link, .entry, article').slice(0, 10).each((i, elem) => {
            const caseData = {
              title: $(elem).find('h2, h3, .title').first().text().trim(),
              court: $(elem).find('.court, .meta').text().trim() || 'Federal Court',
              date_filed: new Date().toISOString(),
              docket_number: $(elem).find('.docket').text().trim() || '',
              summary: $(elem).find('.summary, .excerpt, p').first().text().trim().slice(0, 500),
              url: $(elem).find('a').first().attr('href') || '',
              source: 'Justia',
              data_type: 'case',
              hash: ''
            };
            if (caseData.title) {
              caseData.hash = generateHash(caseData);
              cases.push(caseData);
            }
          });
        }
      } catch (error) {
        console.error(`Error scraping cases from ${source.url}:`, error.message);
      }
    }
    
    return cases;
  },

  // Scrape legal awards
  async scrapeAwards() {
    const sources = [
      'https://www.law.com/nationallawjournal/professional-excellence-awards/',
      'https://www.chambersandpartners.com/awards',
      'https://www.americanlawyermedia.com/awards/'
    ];
    
    const awards = [];
    
    for (const url of sources) {
      try {
        const response = await axios.get(url);
        const $ = cheerio.load(response.data);
        
        $('.award, .winner, article, .entry').slice(0, 10).each((i, elem) => {
          const award = {
            title: $(elem).find('h2, h3, .title').first().text().trim(),
            recipient: $(elem).find('.recipient, .lawyer, .firm').text().trim(),
            organization: $(elem).find('.org, .awarding-body').text().trim() || 'Legal Awards Organization',
            date_awarded: new Date().toISOString(),
            category: $(elem).find('.category, .practice-area').text().trim(),
            description: $(elem).find('.description, .summary, p').first().text().trim().slice(0, 500),
            url: $(elem).find('a').first().attr('href') || url,
            source: new URL(url).hostname,
            data_type: 'award',
            hash: ''
          };
          if (award.title || award.recipient) {
            award.hash = generateHash(award);
            awards.push(award);
          }
        });
      } catch (error) {
        console.error(`Error scraping awards from ${url}:`, error.message);
      }
    }
    
    return awards;
  },

  // Scrape law firms
  async scrapeLawFirms() {
    const sources = [
      'https://www.vault.com/best-companies-to-work-for/law/top-100-law-firms-rankings',
      'https://www.law.com/americanlawyer/am-law-200/'
    ];
    
    const firms = [];
    
    for (const url of sources) {
      try {
        const response = await axios.get(url);
        const $ = cheerio.load(response.data);
        
        $('.firm, .company, tr, .ranking-item').slice(0, 15).each((i, elem) => {
          const firm = {
            name: $(elem).find('.firm-name, .company-name, td:first-child, h3').first().text().trim(),
            ranking: $(elem).find('.rank, .position, td:nth-child(2)').text().trim(),
            location: $(elem).find('.location, .headquarters, td:nth-child(3)').text().trim() || 'United States',
            size: $(elem).find('.size, .attorneys, td:nth-child(4)').text().trim(),
            practice_areas: $(elem).find('.practice-areas, .specialties').text().trim(),
            revenue: $(elem).find('.revenue, td:nth-child(5)').text().trim(),
            website: $(elem).find('a').attr('href') || '',
            source: new URL(url).hostname,
            data_type: 'law_firm',
            hash: ''
          };
          if (firm.name) {
            firm.hash = generateHash(firm);
            firms.push(firm);
          }
        });
      } catch (error) {
        console.error(`Error scraping law firms from ${url}:`, error.message);
      }
    }
    
    return firms;
  },

  // Scrape non-profits
  async scrapeNonProfits() {
    const sources = [
      'https://www.charitynavigator.org/index.cfm?bay=topten.detail&listid=148',
      'https://www.guidestar.org/NonprofitDirectory.aspx'
    ];
    
    const nonprofits = [];
    
    for (const url of sources) {
      try {
        const response = await axios.get(url);
        const $ = cheerio.load(response.data);
        
        $('.nonprofit, .organization, .charity, tr').slice(0, 10).each((i, elem) => {
          const nonprofit = {
            name: $(elem).find('.org-name, .charity-name, td:first-child, h3').first().text().trim(),
            category: $(elem).find('.category, .cause').text().trim() || 'Legal Aid',
            location: $(elem).find('.location, .city-state').text().trim() || 'United States',
            mission: $(elem).find('.mission, .description, p').first().text().trim().slice(0, 500),
            rating: $(elem).find('.rating, .score').text().trim(),
            website: $(elem).find('a').attr('href') || '',
            source: new URL(url).hostname,
            data_type: 'nonprofit',
            hash: ''
          };
          if (nonprofit.name) {
            nonprofit.hash = generateHash(nonprofit);
            nonprofits.push(nonprofit);
          }
        });
      } catch (error) {
        console.error(`Error scraping non-profits from ${url}:`, error.message);
      }
    }
    
    return nonprofits;
  },

  // Scrape legal news
  async scrapeNews() {
    const sources = [
      'https://www.law.com/feed/',
      'https://www.abajournal.com/news/article/rss',
      'https://www.reuters.com/legal/rss'
    ];
    
    const news = [];
    
    for (const url of sources) {
      try {
        const response = await axios.get(url);
        const $ = cheerio.load(response.data, { xmlMode: true });
        
        $('item, entry').slice(0, 10).each((i, elem) => {
          const article = {
            title: $(elem).find('title').text().trim(),
            description: $(elem).find('description, summary').text().trim().slice(0, 500),
            url: $(elem).find('link').text().trim() || $(elem).find('link').attr('href'),
            published_date: $(elem).find('pubDate, published').text().trim() || new Date().toISOString(),
            author: $(elem).find('author, creator').text().trim(),
            category: $(elem).find('category').first().text().trim() || 'Legal News',
            source: new URL(url).hostname,
            data_type: 'news',
            hash: ''
          };
          if (article.title) {
            article.hash = generateHash(article);
            news.push(article);
          }
        });
      } catch (error) {
        console.error(`Error scraping news from ${url}:`, error.message);
      }
    }
    
    return news;
  }
};

// Main scraping function
async function scrapeAll() {
  console.log(`[${new Date().toISOString()}] Starting scraping process...`);
  
  try {
    // Scrape all data types
    const [cases, awards, firms, nonprofits, news] = await Promise.all([
      scrapers.scrapeCases(),
      scrapers.scrapeAwards(),
      scrapers.scrapeLawFirms(),
      scrapers.scrapeNonProfits(),
      scrapers.scrapeNews()
    ]);
    
    // Store in Supabase with deduplication
    const results = {
      cases: 0,
      awards: 0,
      firms: 0,
      nonprofits: 0,
      news: 0
    };
    
    // Insert cases
    for (const item of cases) {
      const { data, error } = await supabase
        .from('scraped_cases')
        .upsert(item, { onConflict: 'hash' });
      if (!error) results.cases++;
    }
    
    // Insert awards
    for (const item of awards) {
      const { data, error } = await supabase
        .from('scraped_awards')
        .upsert(item, { onConflict: 'hash' });
      if (!error) results.awards++;
    }
    
    // Insert law firms
    for (const item of firms) {
      const { data, error } = await supabase
        .from('scraped_law_firms')
        .upsert(item, { onConflict: 'hash' });
      if (!error) results.firms++;
    }
    
    // Insert non-profits
    for (const item of nonprofits) {
      const { data, error } = await supabase
        .from('scraped_nonprofits')
        .upsert(item, { onConflict: 'hash' });
      if (!error) results.nonprofits++;
    }
    
    // Insert news
    for (const item of news) {
      const { data, error } = await supabase
        .from('scraped_news')
        .upsert(item, { onConflict: 'hash' });
      if (!error) results.news++;
    }
    
    console.log(`[${new Date().toISOString()}] Scraping completed:`, results);
    
    // Log to scraping history
    await supabase.from('scraping_logs').insert({
      timestamp: new Date().toISOString(),
      results: results,
      status: 'success'
    });
    
  } catch (error) {
    console.error('Scraping error:', error);
    await supabase.from('scraping_logs').insert({
      timestamp: new Date().toISOString(),
      error: error.message,
      status: 'error'
    });
  }
}

// Schedule hourly scraping
cron.schedule('0 * * * *', () => {
  console.log('Running scheduled scraping...');
  scrapeAll();
});

// Initial scrape on startup
scrapeAll();

console.log('Scraper started. Will run every hour.');