← back to Sdcc Awards

cypressaward/scraper/test-scraper.js

144 lines

require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const axios = require('axios');
const cheerio = require('cheerio');
const crypto = require('crypto');

// Supabase configuration
const supabaseUrl = process.env.SUPABASE_URL;
const supabaseKey = process.env.SUPABASE_ANON_KEY;
const supabase = createClient(supabaseUrl, supabaseKey);

// Test with simple mock data
async function testScraper() {
  console.log('Testing scraper with mock data...');
  
  // Generate test data
  const testCases = [
    {
      hash: crypto.createHash('md5').update(JSON.stringify({title: 'Test Case 1'})).digest('hex'),
      title: 'Smith v. Jones',
      court: 'US Supreme Court',
      date_filed: new Date().toISOString(),
      docket_number: '2024-123',
      summary: 'A landmark case about property rights',
      url: 'https://example.com/case1',
      source: 'Test Source',
      data_type: 'case'
    }
  ];
  
  const testAwards = [
    {
      hash: crypto.createHash('md5').update(JSON.stringify({title: 'Test Award 1'})).digest('hex'),
      title: 'Lawyer of the Year 2024',
      recipient: 'John Doe',
      organization: 'Legal Awards Organization',
      date_awarded: new Date().toISOString(),
      category: 'Litigation',
      description: 'Outstanding achievement in litigation',
      url: 'https://example.com/award1',
      source: 'Test Source',
      data_type: 'award'
    }
  ];
  
  const testFirms = [
    {
      hash: crypto.createHash('md5').update(JSON.stringify({name: 'Test Firm 1'})).digest('hex'),
      name: 'BigLaw & Associates',
      ranking: '#1',
      location: 'New York, NY',
      size: '500+ attorneys',
      practice_areas: 'Corporate, Litigation, IP',
      revenue: '$1B+',
      website: 'https://biglaw.com',
      source: 'Test Source',
      data_type: 'law_firm'
    }
  ];
  
  const testNonprofits = [
    {
      hash: crypto.createHash('md5').update(JSON.stringify({name: 'Test Nonprofit 1'})).digest('hex'),
      name: 'Legal Aid Society',
      category: 'Legal Services',
      location: 'Los Angeles, CA',
      mission: 'Providing free legal services to those in need',
      rating: '4.5/5',
      website: 'https://legalaid.org',
      source: 'Test Source',
      data_type: 'nonprofit'
    }
  ];
  
  const testNews = [
    {
      hash: crypto.createHash('md5').update(JSON.stringify({title: 'Test News 1'})).digest('hex'),
      title: 'Major Legal Development in Technology Law',
      description: 'Courts rule on new AI regulations affecting tech companies',
      url: 'https://example.com/news1',
      published_date: new Date().toISOString(),
      author: 'Jane Reporter',
      category: 'Tech Law',
      source: 'Test Source',
      data_type: 'news'
    }
  ];
  
  // Insert test data into Supabase
  console.log('Inserting test data into Supabase...');
  
  try {
    // Insert cases
    const { data: casesData, error: casesError } = await supabase
      .from('scraped_cases')
      .upsert(testCases, { onConflict: 'hash' });
    console.log('Cases:', casesError ? casesError.message : 'Success');
    
    // Insert awards
    const { data: awardsData, error: awardsError } = await supabase
      .from('scraped_awards')
      .upsert(testAwards, { onConflict: 'hash' });
    console.log('Awards:', awardsError ? awardsError.message : 'Success');
    
    // Insert firms
    const { data: firmsData, error: firmsError } = await supabase
      .from('scraped_law_firms')
      .upsert(testFirms, { onConflict: 'hash' });
    console.log('Firms:', firmsError ? firmsError.message : 'Success');
    
    // Insert nonprofits
    const { data: nonprofitsData, error: nonprofitsError } = await supabase
      .from('scraped_nonprofits')
      .upsert(testNonprofits, { onConflict: 'hash' });
    console.log('Nonprofits:', nonprofitsError ? nonprofitsError.message : 'Success');
    
    // Insert news
    const { data: newsData, error: newsError } = await supabase
      .from('scraped_news')
      .upsert(testNews, { onConflict: 'hash' });
    console.log('News:', newsError ? newsError.message : 'Success');
    
    // Log to scraping history
    await supabase.from('scraping_logs').insert({
      timestamp: new Date().toISOString(),
      results: {
        cases: 1,
        awards: 1,
        firms: 1,
        nonprofits: 1,
        news: 1
      },
      status: 'test_success'
    });
    
    console.log('\nTest completed successfully!');
    console.log('Check your Supabase dashboard to see the data.');
    
  } catch (error) {
    console.error('Test error:', error);
  }
}

testScraper();