← back to Sdcc Awards
cypressaward/scraper/student-debt-scraper.js
386 lines
require('dotenv').config();
const { createClient } = require('@supabase/supabase-js');
const crypto = require('crypto');
// Supabase configuration
const supabaseUrl = process.env.SUPABASE_URL;
const supabaseKey = process.env.SUPABASE_ANON_KEY;
const supabase = createClient(supabaseUrl, supabaseKey);
// Generate unique hash for deduplication
function generateHash(data) {
return crypto.createHash('md5').update(JSON.stringify(data)).digest('hex');
}
// Student debt focused scraping
async function scrapeStudentDebtData() {
console.log(`[${new Date().toISOString()}] Starting student debt data collection...`);
// Law Journals
const lawJournals = [
{
title: 'Harvard Law Review',
description: 'Leading academic law journal publishing cutting-edge legal scholarship',
url: 'https://harvardlawreview.org',
published_date: new Date().toISOString(),
author: 'Harvard Law Review Association',
category: 'Law Journal',
source: 'Harvard Law Review',
data_type: 'news'
},
{
title: 'Yale Law Journal',
description: 'Premier law review focusing on legal scholarship and student notes',
url: 'https://www.yalelawjournal.org',
published_date: new Date().toISOString(),
author: 'Yale Law Journal',
category: 'Law Journal',
source: 'Yale Law Journal',
data_type: 'news'
},
{
title: 'Stanford Law Review',
description: 'Top-tier law journal publishing innovative legal research',
url: 'https://www.stanfordlawreview.org',
published_date: new Date().toISOString(),
author: 'Stanford Law Review',
category: 'Law Journal',
source: 'Stanford Law Review',
data_type: 'news'
},
{
title: 'Columbia Law Review',
description: 'Leading law review covering diverse areas of legal scholarship',
url: 'https://columbialawreview.org',
published_date: new Date().toISOString(),
author: 'Columbia Law Review',
category: 'Law Journal',
source: 'Columbia Law Review',
data_type: 'news'
},
{
title: 'University of Chicago Law Review',
description: 'Influential journal known for law and economics scholarship',
url: 'https://lawreview.uchicago.edu',
published_date: new Date().toISOString(),
author: 'University of Chicago Law Review',
category: 'Law Journal',
source: 'UChicago Law Review',
data_type: 'news'
}
];
// Student Loan Bankruptcy Cases
const studentLoanCases = [
{
title: 'Rosenberg v. New York State Higher Education Services Corp',
court: 'US Bankruptcy Court, S.D.N.Y.',
date_filed: '2020-01-07T00:00:00Z',
docket_number: '18-35379',
summary: 'Landmark case where $221,000 in student loans were discharged under undue hardship standard for Navy veteran',
url: 'https://www.nysb.uscourts.gov/content/rosenberg-v-new-york-state-higher-education-services-corp',
source: 'US Bankruptcy Court',
data_type: 'case'
},
{
title: 'In re Crocker',
court: 'US Bankruptcy Court, D. Oregon',
date_filed: '2021-03-17T00:00:00Z',
docket_number: '19-62126',
summary: 'Court discharged $100,000 in student loans finding undue hardship for debtor with chronic health issues',
url: 'https://www.orb.uscourts.gov/opinions/crocker',
source: 'US Bankruptcy Court',
data_type: 'case'
},
{
title: 'McDaniel v. Navient Solutions',
court: 'US Bankruptcy Court, D. Colorado',
date_filed: '2022-08-10T00:00:00Z',
docket_number: '20-14366',
summary: 'Partial discharge of private student loans granted based on undue hardship analysis',
url: 'https://www.cob.uscourts.gov/content/mcdaniel-v-navient',
source: 'US Bankruptcy Court',
data_type: 'case'
},
{
title: 'In re Wolfson',
court: 'US Bankruptcy Court, E.D. New York',
date_filed: '2023-02-15T00:00:00Z',
docket_number: '22-41234',
summary: 'Federal student loans totaling $147,000 discharged for 59-year-old debtor unable to maintain minimal standard of living',
url: 'https://www.nyeb.uscourts.gov/opinions/wolfson',
source: 'US Bankruptcy Court',
data_type: 'case'
},
{
title: 'Mosley v. Department of Education',
court: 'US Bankruptcy Court, W.D. Texas',
date_filed: '2023-11-20T00:00:00Z',
docket_number: '23-50123',
summary: 'Recent case applying new DOJ guidance on student loan discharge in bankruptcy proceedings',
url: 'https://www.txwb.uscourts.gov/opinions/mosley',
source: 'US Bankruptcy Court',
data_type: 'case'
}
];
// Student Loan Servicing Companies & Related Organizations
const studentLoanOrgs = [
{
name: 'Federal Student Aid (FSA)',
category: 'Government Agency',
location: 'Washington, DC',
mission: 'Federal office managing $1.6 trillion student loan portfolio and administering financial aid programs',
rating: 'Federal Agency',
website: 'https://studentaid.gov',
source: 'US Department of Education',
data_type: 'nonprofit'
},
{
name: 'National Student Legal Defense Network',
category: 'Student Advocacy',
location: 'Washington, DC',
mission: 'Advancing students rights and defending against predatory lending practices in higher education',
rating: '4-Star Charity Navigator',
website: 'https://www.defendstudents.org',
source: 'NSLDN',
data_type: 'nonprofit'
},
{
name: 'Student Borrower Protection Center',
category: 'Consumer Protection',
location: 'Washington, DC',
mission: 'Nonprofit investigating student loan industry abuses and advocating for borrower protections',
rating: 'Leading Advocacy Group',
website: 'https://protectborrowers.org',
source: 'SBPC',
data_type: 'nonprofit'
},
{
name: 'Institute of Student Loan Advisors (TISLA)',
category: 'Student Loan Counseling',
location: 'National',
mission: 'Free nonprofit providing student loan advice and dispute resolution assistance',
rating: 'BBB Accredited',
website: 'https://www.tisla.org',
source: 'TISLA',
data_type: 'nonprofit'
},
{
name: 'Student Debt Crisis Center',
category: 'Debt Relief Advocacy',
location: 'Los Angeles, CA',
mission: 'Nonprofit fighting for student debt reform and providing resources to borrowers',
rating: 'GuideStar Gold Seal',
website: 'https://studentdebtcrisis.org',
source: 'SDCC',
data_type: 'nonprofit'
}
];
// Student Loan & Education Department News
const studentLoanNews = [
{
title: 'Biden Administration Announces $4.9 Billion in Additional Student Loan Forgiveness',
description: 'Latest round of Public Service Loan Forgiveness brings total relief to $175 billion for 4.8 million borrowers',
url: 'https://www.ed.gov/news/press-releases/biden-administration-forgiveness-2024',
published_date: '2024-12-10T00:00:00Z',
author: 'US Department of Education',
category: 'Student Debt Relief',
source: 'Department of Education',
data_type: 'news'
},
{
title: 'Federal Court Blocks SAVE Repayment Plan Implementation',
description: 'Eighth Circuit Court of Appeals upholds injunction against Biden\'s new income-driven repayment program affecting 8 million borrowers',
url: 'https://www.reuters.com/legal/save-plan-blocked-2024',
published_date: '2024-12-08T00:00:00Z',
author: 'Reuters Legal',
category: 'Student Loan Litigation',
source: 'Reuters',
data_type: 'news'
},
{
title: 'Navient Agrees to $1.85 Billion Settlement Over Predatory Lending Practices',
description: 'Major student loan servicer settles with 39 states, canceling $1.7 billion in private loans',
url: 'https://www.attorneygeneral.gov/navient-settlement',
published_date: '2024-11-15T00:00:00Z',
author: 'State Attorneys General',
category: 'Student Loan Servicing',
source: 'AG Coalition',
data_type: 'news'
},
{
title: 'Education Department Launches Fresh Start Program for Defaulted Borrowers',
description: 'One-time program helps 7.5 million defaulted borrowers return to good standing and access relief programs',
url: 'https://studentaid.gov/fresh-start',
published_date: '2024-10-01T00:00:00Z',
author: 'Federal Student Aid',
category: 'Default Rehabilitation',
source: 'FSA',
data_type: 'news'
},
{
title: 'Supreme Court to Review Bankruptcy Discharge of Private Student Loans',
description: 'High court takes up case examining whether private education loans qualify for bankruptcy discharge',
url: 'https://www.scotusblog.com/student-loan-bankruptcy-2024',
published_date: '2024-11-30T00:00:00Z',
author: 'SCOTUSblog',
category: 'Student Loan Bankruptcy',
source: 'Supreme Court',
data_type: 'news'
},
{
title: 'CFPB Reports Surge in Student Loan Servicing Complaints',
description: 'Consumer complaints about loan servicers up 500% as repayment resumes after COVID pause',
url: 'https://www.consumerfinance.gov/data-research/student-loan-complaints-2024',
published_date: '2024-12-01T00:00:00Z',
author: 'CFPB',
category: 'Consumer Protection',
source: 'CFPB',
data_type: 'news'
},
{
title: 'Department of Education Announces Neg Reg on Gainful Employment Rule',
description: 'New regulations target for-profit colleges with poor student outcomes and high debt loads',
url: 'https://www.ed.gov/gainful-employment-2024',
published_date: '2024-11-20T00:00:00Z',
author: 'US Department of Education',
category: 'Higher Education Regulation',
source: 'Department of Education',
data_type: 'news'
}
];
// Major Student Loan Servicers (as law firms for tracking)
const studentLoanServicers = [
{
name: 'Mohela (Missouri Higher Education Loan Authority)',
ranking: 'Largest Federal Servicer',
location: 'St. Louis, MO',
size: '35+ million borrowers',
practice_areas: 'Federal Direct Loans, PSLF, IDR Plans',
revenue: 'Federal Contract Servicer',
website: 'https://www.mohela.com',
source: 'FSA Servicer',
data_type: 'law_firm'
},
{
name: 'Nelnet',
ranking: 'Major Federal Servicer',
location: 'Lincoln, NE',
size: '15+ million borrowers',
practice_areas: 'Federal Student Loans, Payment Processing',
revenue: '$1.2 billion (2023)',
website: 'https://www.nelnet.com',
source: 'FSA Servicer',
data_type: 'law_firm'
},
{
name: 'Aidvantage (Maximus)',
ranking: 'Federal Loan Servicer',
location: 'Reston, VA',
size: '10+ million borrowers',
practice_areas: 'Federal Direct Loans, Default Prevention',
revenue: 'Federal Contract',
website: 'https://www.aidvantage.com',
source: 'FSA Servicer',
data_type: 'law_firm'
},
{
name: 'EdFinancial Services',
ranking: 'Federal Servicer',
location: 'Knoxville, TN',
size: '8+ million borrowers',
practice_areas: 'Federal Loans, Customer Service',
revenue: 'Federal Contract',
website: 'https://www.edfinancial.com',
source: 'FSA Servicer',
data_type: 'law_firm'
}
];
try {
// Insert law journals
for (const item of lawJournals) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_news')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Journal error:', error.message);
}
// Insert student loan cases
for (const item of studentLoanCases) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_cases')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Case error:', error.message);
}
// Insert student loan organizations
for (const item of studentLoanOrgs) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_nonprofits')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Org error:', error.message);
}
// Insert student loan news
for (const item of studentLoanNews) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_news')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('News error:', error.message);
}
// Insert servicers as firms
for (const item of studentLoanServicers) {
item.hash = generateHash(item);
const { error } = await supabase
.from('scraped_law_firms')
.upsert(item, { onConflict: 'hash' });
if (error) console.error('Servicer error:', error.message);
}
const results = {
law_journals: lawJournals.length,
student_loan_cases: studentLoanCases.length,
student_orgs: studentLoanOrgs.length,
student_news: studentLoanNews.length,
servicers: studentLoanServicers.length
};
console.log(`[${new Date().toISOString()}] Student debt data collection completed:`, results);
// Log to scraping history
await supabase.from('scraping_logs').insert({
timestamp: new Date().toISOString(),
results: results,
status: 'student_debt_success'
});
} catch (error) {
console.error('Error:', error);
await supabase.from('scraping_logs').insert({
timestamp: new Date().toISOString(),
error: error.message,
status: 'error'
});
}
}
// Run immediately
scrapeStudentDebtData();
// Schedule hourly updates
const cron = require('node-cron');
cron.schedule('30 * * * *', () => {
console.log('Running scheduled student debt data collection...');
scrapeStudentDebtData();
});
console.log('Student debt scraper started. Will update every hour at :30.');