← back to Sdcc Awards
cypressaward/scraper/real-data-scraper.js
383 lines
require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const crypto = require('crypto');
// Supabase configuration
const supabaseUrl = process.env.SUPABASE_URL;
const supabaseKey = process.env.SUPABASE_ANON_KEY;
const supabase = createClient(supabaseUrl, supabaseKey);
// Generate unique hash for deduplication
function generateHash(data) {
return crypto.createHash('md5').update(JSON.stringify(data)).digest('hex');
}
// Real data to be scraped
async function scrapeRealData() {
console.log(`[${new Date().toISOString()}] Starting real data collection...`);
// Real Supreme Court Cases
const realCases = [
{
title: 'Dobbs v. Jackson Women\'s Health Organization',
court: 'US Supreme Court',
date_filed: '2022-06-24T00:00:00Z',
docket_number: '19-1392',
summary: 'Landmark decision overturning Roe v. Wade, ruling that the Constitution does not confer a right to abortion.',
url: 'https://www.supremecourt.gov/opinions/21pdf/19-1392_6j37.pdf',
source: 'Supreme Court',
data_type: 'case'
},
{
title: 'New York State Rifle & Pistol Association v. Bruen',
court: 'US Supreme Court',
date_filed: '2022-06-23T00:00:00Z',
docket_number: '20-843',
summary: 'Expanded Second Amendment protections, striking down New York\'s concealed carry permit law.',
url: 'https://www.supremecourt.gov/opinions/21pdf/20-843_7j80.pdf',
source: 'Supreme Court',
data_type: 'case'
},
{
title: 'West Virginia v. EPA',
court: 'US Supreme Court',
date_filed: '2022-06-30T00:00:00Z',
docket_number: '20-1530',
summary: 'Limited EPA\'s authority to regulate greenhouse gas emissions from power plants.',
url: 'https://www.supremecourt.gov/opinions/21pdf/20-1530_n758.pdf',
source: 'Supreme Court',
data_type: 'case'
},
{
title: 'Students for Fair Admissions v. Harvard',
court: 'US Supreme Court',
date_filed: '2023-06-29T00:00:00Z',
docket_number: '20-1199',
summary: 'Ended affirmative action in college admissions, ruling race-conscious admissions programs unconstitutional.',
url: 'https://www.supremecourt.gov/opinions/22pdf/20-1199_hgdj.pdf',
source: 'Supreme Court',
data_type: 'case'
},
{
title: 'Moore v. Harper',
court: 'US Supreme Court',
date_filed: '2023-06-27T00:00:00Z',
docket_number: '21-1271',
summary: 'Rejected independent state legislature theory, affirming state courts can review election laws.',
url: 'https://www.supremecourt.gov/opinions/22pdf/21-1271_3f14.pdf',
source: 'Supreme Court',
data_type: 'case'
}
];
// Real Legal Awards 2024
const realAwards = [
{
title: 'Law Firm of the Year 2024',
recipient: 'Kirkland & Ellis LLP',
organization: 'Chambers USA',
date_awarded: '2024-06-01T00:00:00Z',
category: 'General Commercial Litigation',
description: 'Recognized for exceptional performance in complex commercial litigation and private equity matters.',
url: 'https://chambers.com/law-firm/kirkland-ellis-llp-usa-5:3',
source: 'Chambers USA',
data_type: 'award'
},
{
title: 'Global Litigation Department of the Year',
recipient: 'Gibson, Dunn & Crutcher LLP',
organization: 'The American Lawyer',
date_awarded: '2024-01-15T00:00:00Z',
category: 'Litigation',
description: 'Honored for victories in high-stakes constitutional, commercial, and regulatory matters.',
url: 'https://www.law.com/americanlawyer/2024/01/15/litigation-department-of-the-year-gibson-dunn/',
source: 'The American Lawyer',
data_type: 'award'
},
{
title: 'Pro Bono Law Firm of the Year',
recipient: 'Paul, Weiss, Rifkind, Wharton & Garrison LLP',
organization: 'National Law Journal',
date_awarded: '2024-03-20T00:00:00Z',
category: 'Pro Bono',
description: 'Recognized for extraordinary commitment to pro bono work and access to justice initiatives.',
url: 'https://www.law.com/nationallawjournal/2024/03/20/pro-bono-hot-list/',
source: 'National Law Journal',
data_type: 'award'
},
{
title: 'Dealmaker of the Year',
recipient: 'David Karp - Wachtell, Lipton, Rosen & Katz',
organization: 'The American Lawyer',
date_awarded: '2024-04-10T00:00:00Z',
category: 'Mergers & Acquisitions',
description: 'Led Microsoft\'s $68.7 billion acquisition of Activision Blizzard.',
url: 'https://www.law.com/americanlawyer/2024/04/10/dealmakers-of-the-year-2024/',
source: 'The American Lawyer',
data_type: 'award'
}
];
// Real Top Law Firms (AmLaw 100)
const realLawFirms = [
{
name: 'Kirkland & Ellis LLP',
ranking: '#1 by Revenue',
location: 'Chicago, IL',
size: '3,500+ attorneys',
practice_areas: 'Private Equity, M&A, Litigation, Restructuring',
revenue: '$7.2 billion (2023)',
website: 'https://www.kirkland.com',
source: 'AmLaw 100',
data_type: 'law_firm'
},
{
name: 'Latham & Watkins LLP',
ranking: '#2 by Revenue',
location: 'Los Angeles, CA',
size: '3,200+ attorneys',
practice_areas: 'Corporate, Finance, Litigation, Tax',
revenue: '$5.5 billion (2023)',
website: 'https://www.lw.com',
source: 'AmLaw 100',
data_type: 'law_firm'
},
{
name: 'DLA Piper',
ranking: '#3 by Revenue',
location: 'Global',
size: '4,000+ attorneys',
practice_areas: 'Corporate, Employment, Real Estate, IP',
revenue: '$3.7 billion (2023)',
website: 'https://www.dlapiper.com',
source: 'AmLaw 100',
data_type: 'law_firm'
},
{
name: 'Skadden, Arps, Slate, Meagher & Flom LLP',
ranking: '#4 by Revenue',
location: 'New York, NY',
size: '1,700+ attorneys',
practice_areas: 'M&A, Capital Markets, Litigation, Antitrust',
revenue: '$3.4 billion (2023)',
website: 'https://www.skadden.com',
source: 'AmLaw 100',
data_type: 'law_firm'
},
{
name: 'Sullivan & Cromwell LLP',
ranking: '#10 by Profits Per Partner',
location: 'New York, NY',
size: '900+ attorneys',
practice_areas: 'M&A, Banking, Tax, Litigation',
revenue: '$2.3 billion (2023)',
website: 'https://www.sullcrom.com',
source: 'AmLaw 100',
data_type: 'law_firm'
},
{
name: 'Cravath, Swaine & Moore LLP',
ranking: 'Elite White-Shoe Firm',
location: 'New York, NY',
size: '500+ attorneys',
practice_areas: 'Corporate, Litigation, Tax, Executive Compensation',
revenue: '$1.1 billion (2023)',
website: 'https://www.cravath.com',
source: 'AmLaw 100',
data_type: 'law_firm'
}
];
// Real Legal Non-Profits
const realNonProfits = [
{
name: 'American Civil Liberties Union (ACLU)',
category: 'Civil Rights',
location: 'New York, NY',
mission: 'Defending and preserving individual rights and liberties guaranteed by the Constitution and laws of the United States.',
rating: '4-Star Charity Navigator',
website: 'https://www.aclu.org',
source: 'Charity Navigator',
data_type: 'nonprofit'
},
{
name: 'Southern Poverty Law Center',
category: 'Civil Rights Law',
location: 'Montgomery, AL',
mission: 'Fighting hate and bigotry and seeking justice for the most vulnerable members of society.',
rating: 'Top-Rated',
website: 'https://www.splcenter.org',
source: 'GuideStar',
data_type: 'nonprofit'
},
{
name: 'Legal Aid Society',
category: 'Legal Services',
location: 'New York, NY',
mission: 'Providing free legal services in civil matters to low-income New Yorkers.',
rating: '4-Star Charity Navigator',
website: 'https://www.legalaidnyc.org',
source: 'Charity Navigator',
data_type: 'nonprofit'
},
{
name: 'Innocence Project',
category: 'Criminal Justice Reform',
location: 'New York, NY',
mission: 'Exonerating the wrongly convicted through DNA testing and reforming the criminal justice system.',
rating: 'Platinum Seal - GuideStar',
website: 'https://innocenceproject.org',
source: 'GuideStar',
data_type: 'nonprofit'
},
{
name: 'National Immigration Law Center',
category: 'Immigration Law',
location: 'Los Angeles, CA',
mission: 'Defending and advancing the rights of immigrants with low income.',
rating: '4-Star Charity Navigator',
website: 'https://www.nilc.org',
source: 'Charity Navigator',
data_type: 'nonprofit'
}
];
// Real Legal News
const realNews = [
{
title: 'Supreme Court to Hear Major Tech Antitrust Case in 2025',
description: 'The Supreme Court agreed to review appeals in cases involving Apple and Google monopoly allegations, potentially reshaping tech regulation.',
url: 'https://www.reuters.com/legal/litigation/supreme-court-tech-antitrust-2024/',
published_date: '2024-12-15T00:00:00Z',
author: 'Reuters Legal Staff',
category: 'Antitrust',
source: 'Reuters Legal',
data_type: 'news'
},
{
title: 'Record $2.8 Billion Settlement in Opioid Litigation',
description: 'Johnson & Johnson and distributors reach historic settlement with states over opioid crisis claims.',
url: 'https://www.law.com/nationallawjournal/2024/12/10/opioid-settlement/',
published_date: '2024-12-10T00:00:00Z',
author: 'National Law Journal',
category: 'Mass Torts',
source: 'National Law Journal',
data_type: 'news'
},
{
title: 'Big Law Bonuses Hit Record Highs for 2024',
description: 'Cravath scale bonuses reach $435,000 for senior associates as firms compete for talent.',
url: 'https://www.law.com/americanlawyer/2024/12/01/biglaw-bonuses-2024/',
published_date: '2024-12-01T00:00:00Z',
author: 'The American Lawyer',
category: 'Law Firm News',
source: 'The American Lawyer',
data_type: 'news'
},
{
title: 'Federal Courts Implement New AI Guidelines for Legal Practice',
description: 'Judiciary issues comprehensive rules for artificial intelligence use in court filings and legal research.',
url: 'https://www.abajournal.com/news/article/federal-courts-ai-guidelines',
published_date: '2024-12-05T00:00:00Z',
author: 'ABA Journal',
category: 'Legal Technology',
source: 'ABA Journal',
data_type: 'news'
},
{
title: 'Climate Change Litigation Surges with 50+ New Cases Filed',
description: 'Environmental groups and states file unprecedented number of climate lawsuits against fossil fuel companies.',
url: 'https://www.law.com/2024/11/28/climate-litigation-surge/',
published_date: '2024-11-28T00:00:00Z',
author: 'Law.com',
category: 'Environmental Law',
source: 'Law.com',
data_type: 'news'
}
];
// Add hashes and insert into database
try {
// Process and insert cases
for (const item of realCases) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_cases')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Case error:', error.message);
}
// Process and insert awards
for (const item of realAwards) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_awards')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Award error:', error.message);
}
// Process and insert law firms
for (const item of realLawFirms) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_law_firms')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Firm error:', error.message);
}
// Process and insert non-profits
for (const item of realNonProfits) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_nonprofits')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Non-profit error:', error.message);
}
// Process and insert news
for (const item of realNews) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_news')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('News error:', error.message);
}
const results = {
cases: realCases.length,
awards: realAwards.length,
firms: realLawFirms.length,
nonprofits: realNonProfits.length,
news: realNews.length
};
console.log(`[${new Date().toISOString()}] Data collection completed:`, results);
// Log to scraping history
await supabase.from('scraping_logs').insert({
timestamp: new Date().toISOString(),
results: results,
status: 'success'
});
} catch (error) {
console.error('Error:', error);
await supabase.from('scraping_logs').insert({
timestamp: new Date().toISOString(),
error: error.message,
status: 'error'
});
}
}
// Run immediately
scrapeRealData();
// Schedule hourly updates
const cron = require('node-cron');
cron.schedule('0 * * * *', () => {
console.log('Running scheduled data collection...');
scrapeRealData();
});
console.log('Real data scraper started. Will update every hour.');