← back to Norma
agents/price-agent/lib/html-parser.js
753 lines
/**
* HTML Parsers for California Higher Education Cost Pages
*
* Cheerio-based parsers that extract tuition, fees, room & board, and other
* cost-of-attendance data from UC, CSU, and federal financial aid web pages.
*
* All parsers are defensive — they return null for fields that cannot be
* found or parsed, and never throw.
*
* Usage:
* const { parseUCCostsPage } = require('./html-parser');
* const data = parseUCCostsPage(html, 'UC Berkeley');
*/
const cheerio = require('cheerio');
// ---------------------------------------------------------------------------
// Dollar parsing helper
// ---------------------------------------------------------------------------
/**
* Extract a numeric dollar amount from a text string.
* Handles formats like "$14,312", "14312", "$14,312.50", "$ 14,312".
* Rounds to nearest whole dollar.
*
* @param {string|null|undefined} text
* @returns {number|null}
*/
function parseDollar(text) {
if (text == null || typeof text !== 'string') return null;
// Remove whitespace, commas, and optional leading $
const cleaned = text.replace(/\s/g, '').replace(/,/g, '');
const match = cleaned.match(/\$?([\d]+(?:\.[\d]+)?)/);
if (!match) return null;
const value = parseFloat(match[1]);
if (isNaN(value)) return null;
return Math.round(value);
}
/**
* Try to extract an academic year string (e.g. "2025-2026", "2025-26")
* from the page content.
*
* @param {string} html
* @returns {string|null}
*/
function extractAcademicYear(html) {
if (!html) return null;
// Match patterns like "2025-2026", "2025-26", "2025-2026" (en-dash)
const patterns = [
/20\d{2}\s*[-\u2013\u2014]\s*20\d{2}/,
/20\d{2}\s*[-\u2013\u2014]\s*\d{2}(?!\d)/,
];
for (const pattern of patterns) {
const match = html.match(pattern);
if (match) {
return match[0].replace(/\s/g, '').replace(/[\u2013\u2014]/g, '-');
}
}
return null;
}
// ---------------------------------------------------------------------------
// Row-matching helpers
// ---------------------------------------------------------------------------
/**
* Search for a table row containing a label that matches one of the given
* keywords, then extract the dollar value from the same or adjacent cell.
*
* @param {object} $ - Cheerio instance
* @param {string[]} keywords - Array of lowercase keywords to match
* @param {string} [scope] - Optional CSS selector to narrow search
* @returns {number|null}
*/
function findDollarInTable($, keywords, scope) {
const tables = scope ? $(scope).find('table') : $('table');
let result = null;
tables.each(function () {
if (result !== null) return;
$(this)
.find('tr')
.each(function () {
if (result !== null) return;
const rowText = $(this).text().toLowerCase();
const matched = keywords.some((kw) => rowText.includes(kw));
if (!matched) return;
// Try to find dollar value in td cells (prefer later cells as they
// often hold the value while earlier cells hold the label)
const cells = $(this).find('td, th');
for (let i = cells.length - 1; i >= 0; i--) {
const cellText = $(cells[i]).text().trim();
const val = parseDollar(cellText);
if (val !== null && val > 0) {
result = val;
return;
}
}
});
});
return result;
}
/**
* Search for a dollar amount near a label in any element (not just tables).
* Useful for pages that use definition lists, divs, or spans instead of tables.
*
* @param {object} $ - Cheerio instance
* @param {string[]} keywords - Array of lowercase keywords to match
* @returns {number|null}
*/
function findDollarNearLabel($, keywords) {
let result = null;
// Strategy 1: Look in table rows
result = findDollarInTable($, keywords);
if (result !== null) return result;
// Strategy 2: Look in dt/dd pairs (definition lists)
$('dt, .label, .item-label').each(function () {
if (result !== null) return;
const labelText = $(this).text().toLowerCase();
const matched = keywords.some((kw) => labelText.includes(kw));
if (!matched) return;
// Check the next sibling (dd or value element)
const next = $(this).next();
if (next.length) {
const val = parseDollar(next.text());
if (val !== null && val > 0) {
result = val;
}
}
});
if (result !== null) return result;
// Strategy 3: Look in list items that contain both label and dollar amount
$('li, .cost-item, .row, .flex').each(function () {
if (result !== null) return;
const text = $(this).text().toLowerCase();
const matched = keywords.some((kw) => text.includes(kw));
if (!matched) return;
const val = parseDollar($(this).text());
if (val !== null && val > 0) {
result = val;
}
});
return result;
}
// ---------------------------------------------------------------------------
// UC Cost Page Parser
// ---------------------------------------------------------------------------
/**
* Parse a UC campus financial aid / cost-of-attendance page.
*
* UC pages typically present a table with line items for:
* - Tuition / Student Services Fee
* - Room & Board (on-campus / off-campus)
* - Books and Supplies
* - Personal / Transportation
* - Total Cost of Attendance
*
* @param {string} html - Raw HTML of the page
* @param {string} campusName - Name of the campus (for logging)
* @returns {object} Parsed cost data with null for missing fields
*/
function parseUCCostsPage(html, campusName) {
const result = {
tuition_in_state: null,
mandatory_fees: null,
tuition_and_fees: null,
room_board_on_campus: null,
room_board_off_campus: null,
books_supplies: null,
personal_expenses: null,
transportation: null,
other_expenses: null,
coa_on_campus: null,
coa_off_campus: null,
academic_year: null,
};
if (!html) {
console.warn('[html-parser] Empty HTML for ' + (campusName || 'unknown UC campus'));
return result;
}
let $;
try {
$ = cheerio.load(html);
} catch (err) {
console.warn(
'[html-parser] Failed to parse HTML for ' + campusName + ': ' + err.message
);
return result;
}
// Extract academic year
result.academic_year = extractAcademicYear(html);
// Tuition (UC calls it "Tuition" or "Tuition and Student Services Fee")
result.tuition_in_state = findDollarNearLabel($, [
'tuition and student services fee',
'tuition & student services fee',
'tuition and fees',
'tuition & fees',
'tuition',
]);
// Mandatory fees (sometimes broken out separately)
result.mandatory_fees = findDollarNearLabel($, [
'student services fee',
'mandatory fees',
'campus fees',
'campus-based fees',
]);
// Combined tuition + fees if listed as one line
result.tuition_and_fees = findDollarNearLabel($, [
'tuition and fees total',
'total tuition and fees',
'total fees',
]);
// If we didn't get a separate tuition_and_fees but have tuition, use that
if (result.tuition_and_fees == null && result.tuition_in_state != null) {
result.tuition_and_fees = result.tuition_in_state;
}
// Room & Board
result.room_board_on_campus = findDollarNearLabel($, [
'room and board',
'room & board',
'housing and food',
'housing & food',
'housing and meals',
]);
// Off-campus room & board (sometimes listed separately)
result.room_board_off_campus = findDollarNearLabel($, [
'off-campus room and board',
'off-campus housing',
'off campus room',
'living with parent',
]);
// Books & Supplies
result.books_supplies = findDollarNearLabel($, [
'books and supplies',
'books & supplies',
'books/supplies',
'textbooks',
]);
// Personal expenses
result.personal_expenses = findDollarNearLabel($, [
'personal expenses',
'personal',
'miscellaneous',
]);
// Transportation
result.transportation = findDollarNearLabel($, [
'transportation',
'travel',
]);
// Other expenses (catch-all)
if (result.personal_expenses == null && result.transportation == null) {
result.other_expenses = findDollarNearLabel($, [
'other expenses',
'other costs',
'additional expenses',
]);
}
// Total COA
result.coa_on_campus = findDollarNearLabel($, [
'total cost of attendance',
'total estimated cost',
'total budget',
'total cost',
'estimated total',
]);
// Off-campus total
result.coa_off_campus = findDollarNearLabel($, [
'off-campus total',
'off campus total',
'total off-campus',
]);
// Sanity checks — log warnings for unexpected values
if (result.tuition_in_state != null) {
if (result.tuition_in_state < 1000 || result.tuition_in_state > 100000) {
console.warn(
'[html-parser] Suspicious tuition for ' + campusName + ': $' + result.tuition_in_state
);
}
} else {
console.warn(
'[html-parser] Could not find tuition on ' + campusName + ' costs page.'
);
}
if (result.coa_on_campus != null && result.tuition_in_state != null) {
if (result.coa_on_campus < result.tuition_in_state) {
console.warn(
'[html-parser] COA ($' + result.coa_on_campus + ') < tuition ($' + result.tuition_in_state + ') ' +
'for ' + campusName + ' — likely parsing error.'
);
}
}
return result;
}
// ---------------------------------------------------------------------------
// CSU Cost Page Parser
// ---------------------------------------------------------------------------
/**
* Parse a CSU costs and fees page.
*
* CSU pages (both central and campus-specific) typically break down costs as:
* - Tuition Fee (systemwide)
* - Campus-Based Fees (varies by campus)
* - Room and Board
* - Books and Supplies
* - Personal / Transportation
*
* CSU also distinguishes between undergraduate and graduate, and
* credential programs. We focus on undergraduate.
*
* @param {string} html - Raw HTML of the page
* @param {string} campusName - Name of the campus (for logging)
* @returns {object} Parsed cost data with null for missing fields
*/
function parseCSUCostsPage(html, campusName) {
const result = {
tuition_in_state: null,
campus_fees: null,
tuition_and_fees: null,
room_board_on_campus: null,
room_board_off_campus: null,
books_supplies: null,
personal_expenses: null,
transportation: null,
other_expenses: null,
coa_on_campus: null,
coa_off_campus: null,
nonresident_fee: null,
academic_year: null,
};
if (!html) {
console.warn('[html-parser] Empty HTML for ' + (campusName || 'unknown CSU campus'));
return result;
}
let $;
try {
$ = cheerio.load(html);
} catch (err) {
console.warn(
'[html-parser] Failed to parse HTML for ' + campusName + ': ' + err.message
);
return result;
}
result.academic_year = extractAcademicYear(html);
// CSU Systemwide Tuition Fee
result.tuition_in_state = findDollarNearLabel($, [
'systemwide tuition fee',
'tuition fee',
'basic tuition',
'state university fee',
]);
// Campus-based fees
result.campus_fees = findDollarNearLabel($, [
'campus-based fees',
'campus fees',
'mandatory campus fees',
'associated students fee',
]);
// Total tuition + fees
result.tuition_and_fees = findDollarNearLabel($, [
'total tuition and fees',
'tuition and fees',
'tuition & fees',
'total fees',
]);
// If we have tuition and campus fees but not combined, sum them
if (
result.tuition_and_fees == null &&
result.tuition_in_state != null &&
result.campus_fees != null
) {
result.tuition_and_fees = result.tuition_in_state + result.campus_fees;
}
// Non-resident supplemental fee
result.nonresident_fee = findDollarNearLabel($, [
'nonresident tuition',
'non-resident tuition',
'nonresident supplemental',
'non-resident fee',
'out-of-state fee',
]);
// Room & Board
result.room_board_on_campus = findDollarNearLabel($, [
'room and board',
'room & board',
'housing and meals',
'housing and food',
]);
result.room_board_off_campus = findDollarNearLabel($, [
'off-campus room',
'off campus room',
'off-campus housing',
]);
// Books & Supplies
result.books_supplies = findDollarNearLabel($, [
'books and supplies',
'books & supplies',
'books/supplies',
]);
// Personal expenses
result.personal_expenses = findDollarNearLabel($, [
'personal expenses',
'personal',
]);
// Transportation
result.transportation = findDollarNearLabel($, [
'transportation',
'travel',
]);
// Other expenses
result.other_expenses = findDollarNearLabel($, [
'other expenses',
'other costs',
]);
// Total COA
result.coa_on_campus = findDollarNearLabel($, [
'total cost of attendance',
'total estimated budget',
'total estimated cost',
'total cost',
'estimated total',
]);
result.coa_off_campus = findDollarNearLabel($, [
'off-campus total',
'total off-campus',
]);
// Sanity checks
if (result.tuition_in_state != null) {
// CSU systemwide tuition is typically $5,000-$8,000
if (result.tuition_in_state < 1000 || result.tuition_in_state > 50000) {
console.warn(
'[html-parser] Suspicious CSU tuition for ' + campusName + ': $' + result.tuition_in_state
);
}
} else {
console.warn(
'[html-parser] Could not find tuition on ' + campusName + ' CSU costs page.'
);
}
return result;
}
// ---------------------------------------------------------------------------
// Federal Interest Rates Page Parser
// ---------------------------------------------------------------------------
/**
* Parse the studentaid.gov federal loan interest rates page.
*
* The page typically has a table with columns like:
* Loan Type | Borrower Type | Fixed Interest Rate | Origination Fee
*
* @param {string} html - Raw HTML from studentaid.gov interest rates page
* @returns {object[]} Array of rate objects, or empty array on failure
*/
function parseFederalRatesPage(html) {
if (!html) {
console.warn('[html-parser] Empty HTML for federal rates page.');
return [];
}
let $;
try {
$ = cheerio.load(html);
} catch (err) {
console.warn(
'[html-parser] Failed to parse federal rates HTML: ' + err.message
);
return [];
}
const rates = [];
const effectiveDate = extractAcademicYear(html);
// Try to find the main rates table
$('table').each(function () {
const headers = [];
$(this)
.find('thead th, thead td, tr:first-child th, tr:first-child td')
.each(function () {
headers.push($(this).text().trim().toLowerCase());
});
// Identify column indexes
const loanTypeIdx = headers.findIndex(
function (h) {
return h.includes('loan type') || h.includes('loan') || h.includes('type');
}
);
const rateIdx = headers.findIndex(
function (h) {
return h.includes('interest rate') || h.includes('fixed rate') || h.includes('rate');
}
);
const feeIdx = headers.findIndex(
function (h) {
return h.includes('origination fee') || h.includes('loan fee') || h.includes('fee');
}
);
// Skip tables that don't look like rate tables
if (rateIdx === -1) return;
$(this)
.find('tbody tr, tr')
.each(function (rowIdx) {
// Skip header row if it's in the same selector
if (rowIdx === 0 && headers.length > 0) return;
const cells = [];
$(this)
.find('td, th')
.each(function () {
cells.push($(this).text().trim());
});
if (cells.length < 2) return;
const rateType = loanTypeIdx >= 0 ? cells[loanTypeIdx] : cells[0];
const rateText = rateIdx >= 0 ? cells[rateIdx] : cells[1];
const feeText = feeIdx >= 0 ? cells[feeIdx] : null;
// Parse interest rate (expect format like "5.50%" or "5.50")
const interestRate = parsePercentage(rateText);
const originationFee = feeText ? parsePercentage(feeText) : null;
if (rateType && interestRate !== null) {
rates.push({
rate_type: normalizeRateType(rateType),
rate_type_raw: rateType,
interest_rate: interestRate,
origination_fee: originationFee,
effective_date: effectiveDate,
});
}
});
});
if (rates.length === 0) {
console.warn(
'[html-parser] Could not extract any interest rates from federal rates page.'
);
// Fallback: try to find rates in text content outside tables
const fallbackRates = extractRatesFromText($);
if (fallbackRates.length > 0) {
return fallbackRates;
}
}
return rates;
}
/**
* Parse a percentage string like "5.50%", "5.50", "1.057%".
*
* @param {string|null} text
* @returns {number|null} The percentage as a float (e.g. 5.50 for "5.50%")
*/
function parsePercentage(text) {
if (!text) return null;
const cleaned = text.replace(/\s/g, '').replace(/%/g, '');
const match = cleaned.match(/([\d]+(?:\.[\d]+)?)/);
if (!match) return null;
const val = parseFloat(match[1]);
return isNaN(val) ? null : val;
}
/**
* Normalize a loan type string to a canonical form.
*
* @param {string} raw
* @returns {string}
*/
function normalizeRateType(raw) {
const lower = raw.toLowerCase();
if (lower.includes('direct subsidized') || lower.includes('subsidized stafford')) {
return 'direct_subsidized';
}
if (lower.includes('direct unsubsidized') && lower.includes('undergraduate')) {
return 'direct_unsubsidized_undergrad';
}
if (lower.includes('direct unsubsidized') && lower.includes('graduate')) {
return 'direct_unsubsidized_grad';
}
if (lower.includes('direct unsubsidized') || lower.includes('unsubsidized stafford')) {
return 'direct_unsubsidized';
}
if (lower.includes('direct plus') || lower.includes('parent plus') || lower.includes('grad plus')) {
return 'direct_plus';
}
if (lower.includes('perkins')) {
return 'perkins';
}
if (lower.includes('consolidation')) {
return 'consolidation';
}
// Return cleaned-up version if no match
return lower.replace(/[^a-z0-9_]/g, '_').replace(/_+/g, '_').replace(/^_|_$/g, '');
}
/**
* Fallback: try to extract interest rates from body text when no table is found.
* Looks for patterns like "Direct Subsidized Loans: 5.50%".
*
* @param {object} $
* @returns {object[]}
*/
function extractRatesFromText($) {
const rates = [];
const bodyText = $('body').text();
// Pattern: loan type description followed by a percentage
const loanPatterns = [
{
pattern: /direct\s+subsidized[^:]*?:\s*([\d.]+)\s*%/gi,
type: 'direct_subsidized',
},
{
pattern: /direct\s+unsubsidized[^:]*?undergraduate[^:]*?:\s*([\d.]+)\s*%/gi,
type: 'direct_unsubsidized_undergrad',
},
{
pattern: /direct\s+unsubsidized[^:]*?graduate[^:]*?:\s*([\d.]+)\s*%/gi,
type: 'direct_unsubsidized_grad',
},
{
pattern: /direct\s+plus[^:]*?:\s*([\d.]+)\s*%/gi,
type: 'direct_plus',
},
{
pattern: /parent\s+plus[^:]*?:\s*([\d.]+)\s*%/gi,
type: 'direct_plus',
},
];
for (const item of loanPatterns) {
const match = item.pattern.exec(bodyText);
if (match) {
const rate = parseFloat(match[1]);
if (!isNaN(rate) && rate > 0 && rate < 30) {
rates.push({
rate_type: item.type,
rate_type_raw: match[0].trim(),
interest_rate: rate,
origination_fee: null,
effective_date: extractAcademicYear(bodyText),
});
}
}
}
return rates;
}
// ---------------------------------------------------------------------------
// Generic cost page parser (auto-detect format)
// ---------------------------------------------------------------------------
/**
* Attempt to parse any cost-of-attendance page by trying UC and CSU
* parsers and returning whichever extracts more data.
*
* @param {string} html
* @param {string} schoolName
* @param {string} [system] - 'uc', 'csu', or undefined for auto-detect
* @returns {object}
*/
function parseCostsPage(html, schoolName, system) {
if (system === 'uc') return parseUCCostsPage(html, schoolName);
if (system === 'csu') return parseCSUCostsPage(html, schoolName);
// Auto-detect: try both and pick the one with more non-null fields
const ucResult = parseUCCostsPage(html, schoolName);
const csuResult = parseCSUCostsPage(html, schoolName);
const ucCount = Object.values(ucResult).filter(function (v) { return v != null; }).length;
const csuCount = Object.values(csuResult).filter(function (v) { return v != null; }).length;
if (csuCount > ucCount) return csuResult;
return ucResult;
}
module.exports = {
parseDollar,
parsePercentage,
extractAcademicYear,
parseUCCostsPage,
parseCSUCostsPage,
parseFederalRatesPage,
parseCostsPage,
};