← back to StudentLoanTracker

src/lib/file-intel/index.ts

610 lines

/**
 * File Intelligence Library
 *
 * Client-side file parsing and classification for student loan documents.
 * All processing happens in the browser — files never leave the user's device.
 *
 * Supported formats:
 *   - StudentAid.gov "My Aid Data" TXT export
 *   - CSV/spreadsheet loan data
 *   - PDF documents (servicer statements, ECF results, 1098-E)
 *   - Any text file with loan-related content
 */

// ── Types ──────────────────────────────────────────────────────────

export interface ParsedLoan {
  name: string;
  loanType: string;
  balance: number;
  interestRate: number;
  servicer: string;
  disbursementDate: string;
  status: string;
  outstandingInterest: number;
}

export type DocumentType =
  | 'studentaid_txt'
  | 'csv_loans'
  | 'ecf_pdf'
  | 'servicer_statement'
  | '1098e'
  | 'fsa_letter'
  | 'unknown';

export interface ClassificationResult {
  type: DocumentType;
  confidence: number; // 0-1
  label: string;      // Human-readable description
}

export interface ParseResult {
  classification: ClassificationResult;
  loans: ParsedLoan[];
  pslfPayments?: number;
  rawText: string;
  warnings: string[];
}

// ── Classifier ─────────────────────────────────────────────────────

const SIGNATURES: { type: DocumentType; patterns: RegExp[]; label: string; weight: number }[] = [
  {
    type: 'studentaid_txt',
    patterns: [
      /Federal Student Aid/i,
      /NSLDS/i,
      /My Federal Student Aid Data/i,
      /Loan\s+Type.*Direct/i,
      /Outstanding\s+Principal\s+Balance/i,
      /Loan\s+Status.*In\s+Repayment/i,
    ],
    label: 'StudentAid.gov Data Export',
    weight: 0.25,
  },
  {
    type: 'ecf_pdf',
    patterns: [
      /Public Service Loan Forgiveness/i,
      /Employment Certification/i,
      /ECF/,
      /qualifying\s+payment/i,
      /120\s+payments?/i,
    ],
    label: 'PSLF Employment Certification Form',
    weight: 0.25,
  },
  {
    type: 'servicer_statement',
    patterns: [
      /Monthly\s+Statement/i,
      /Payment\s+Due/i,
      /Account\s+Summary/i,
      /MOHELA|Nelnet|Aidvantage|Edfinancial|Great\s+Lakes|FedLoan/i,
      /Minimum\s+Payment/i,
      /Current\s+Balance/i,
    ],
    label: 'Loan Servicer Statement',
    weight: 0.2,
  },
  {
    type: '1098e',
    patterns: [
      /1098-?E/i,
      /Student\s+Loan\s+Interest\s+Statement/i,
      /Interest\s+(Received|Paid).*\$/i,
    ],
    label: '1098-E Interest Statement',
    weight: 0.35,
  },
  {
    type: 'fsa_letter',
    patterns: [
      /Federal\s+Student\s+Aid/i,
      /Income-?Driven\s+Repayment/i,
      /IDR\s+(Application|Recertification)/i,
      /SAVE\s+Plan|IBR|PAYE|ICR/i,
    ],
    label: 'FSA Letter / IDR Notice',
    weight: 0.2,
  },
  {
    type: 'csv_loans',
    patterns: [
      /balance.*rate|rate.*balance/i,
      /loan.*type.*balance/i,
      /,\d+\.?\d*,\d+\.?\d*/,  // CSV-like numeric fields
    ],
    label: 'CSV Loan Data',
    weight: 0.35,
  },
];

/**
 * Classify a document by analyzing its text content.
 * Uses weighted pattern matching with confidence scoring.
 */
export function classifyDocument(text: string): ClassificationResult {
  let bestType: DocumentType = 'unknown';
  let bestScore = 0;
  let bestLabel = 'Unknown Document';

  for (const sig of SIGNATURES) {
    const matchCount = sig.patterns.filter((p) => p.test(text)).length;
    const score = (matchCount / sig.patterns.length) * sig.weight + (matchCount > 0 ? 0.3 : 0);

    if (score > bestScore) {
      bestScore = score;
      bestType = sig.type;
      bestLabel = sig.label;
    }
  }

  return {
    type: bestType,
    confidence: Math.min(1, bestScore),
    label: bestScore > 0.2 ? bestLabel : 'Unknown Document',
  };
}

// ── Parsers ────────────────────────────────────────────────────────

/**
 * Parse StudentAid.gov "My Aid Data" TXT export.
 *
 * Real format: each line is `"Field Name:Value",`
 * Loan blocks start with `Loan Type Code:XX` (not `Type Code:` which is aggregate totals).
 * Servicer is in `Loan Contact Name:` within the contact sub-block after each loan.
 *
 * State machine: HEADER → AGGREGATE → LOAN → CONTACT → LOAN → ...
 */
export function parseStudentAidTXT(text: string): { loans: ParsedLoan[]; warnings: string[] } {
  const loans: ParsedLoan[] = [];
  const warnings: string[] = [];
  const lines = text.split('\n');

  // Parse "Field Name:Value", lines into key-value pairs
  function parseLine(raw: string): { key: string; value: string } | null {
    const trimmed = raw.trim();
    // Strip surrounding quotes and trailing comma
    const match = trimmed.match(/^"?([^"]*)"?,?\s*$/);
    if (!match) return null;
    const content = match[1];
    const colonIdx = content.indexOf(':');
    if (colonIdx < 0) return null;
    return {
      key: content.substring(0, colonIdx).trim(),
      value: content.substring(colonIdx + 1).trim(),
    };
  }

  let current: Partial<ParsedLoan> | null = null;
  let inLoanBlock = false;
  let schoolName = '';

  for (const line of lines) {
    const kv = parseLine(line);
    if (!kv) continue;

    const { key, value } = kv;
    const k = key.toLowerCase();

    // Detect start of a new LOAN block (not aggregate "Type Code")
    if (key === 'Loan Type Code') {
      // Save previous loan
      if (current && inLoanBlock) {
        loans.push(finalizeLoan(current));
      }
      current = { loanType: detectLoanTypeCode(value) };
      inLoanBlock = true;
      schoolName = '';
      continue;
    }

    // Skip aggregate "Type Code" lines (no "Loan" prefix)
    if (key === 'Type Code' || key === 'Type Description') continue;

    if (!inLoanBlock || !current) continue;

    // ── Loan fields ──
    if (key === 'Loan Type Description') {
      current.name = value;
    } else if (key === 'Loan Attending School Name') {
      schoolName = value;
    } else if (key === 'Loan Date') {
      current.disbursementDate = value;
    } else if (key === 'Loan Outstanding Principal Balance') {
      current.balance = parseDollar(value);
    } else if (key === 'Loan Outstanding Interest Balance') {
      current.outstandingInterest = parseDollar(value);
    } else if (key === 'Loan Interest Rate' || key === 'Loan Actual Interest Rate') {
      if (!current.interestRate) { // Use first one encountered
        current.interestRate = parsePercent(value);
      }
    } else if (key === 'Current Loan Status Description') {
      current.status = value;
    } else if (k === 'loan status description' && !current.status) {
      current.status = value;
    } else if (key === 'Loan Contact Name') {
      current.servicer = value;
    } else if (key === 'Loan Repayment Plan Type Code Description') {
      // Append plan info to name for clarity
      if (current.name && !current.name.includes('(')) {
        const school = schoolName ? ` (${titleCase(schoolName)})` : '';
        current.name = current.name + school;
      }
    }
  }

  // Push last loan
  if (current && inLoanBlock) {
    loans.push(finalizeLoan(current));
  }

  // If the real format wasn't detected, try fallback simple parsing
  if (loans.length === 0) {
    return parseSimpleKeyValue(text);
  }

  const missingBalance = loans.filter((l) => l.balance === 0);
  if (missingBalance.length > 0) {
    warnings.push(`${missingBalance.length} loan(s) have $0 balance — verify these are correct.`);
  }

  return { loans, warnings };
}

/**
 * Fallback parser for non-NSLDS text files with simple Key: Value format.
 */
function parseSimpleKeyValue(text: string): { loans: ParsedLoan[]; warnings: string[] } {
  const loans: ParsedLoan[] = [];
  const warnings: string[] = [];
  const lines = text.split('\n');
  let current: Partial<ParsedLoan> | null = null;

  for (const line of lines) {
    const trimmed = line.trim();
    if (!trimmed) continue;

    if (/^(Direct|FFEL|Perkins|Federal)/i.test(trimmed)) {
      if (current?.name) loans.push(finalizeLoan(current));
      current = { name: trimmed, loanType: detectLoanType(trimmed) };
      continue;
    }

    if (!current) continue;

    const kvMatch = trimmed.match(/^([^:]+?):\s*(.+)$/);
    if (kvMatch) {
      const k = kvMatch[1].trim().toLowerCase();
      const v = kvMatch[2].trim();
      if (k.includes('balance') || k.includes('principal')) current.balance = parseDollar(v);
      else if (k.includes('interest rate') || k.includes('rate')) current.interestRate = parsePercent(v);
      else if (k.includes('servicer')) current.servicer = v;
      else if (k.includes('status')) current.status = v;
      else if (k.includes('disburs') || k.includes('loan date')) current.disbursementDate = v;
    }
  }

  if (current?.name) loans.push(finalizeLoan(current));

  if (loans.length === 0) {
    warnings.push('No loan records found. The file format may not be recognized.');
  }
  return { loans, warnings };
}

/** Map NSLDS type codes to human-readable types */
function detectLoanTypeCode(code: string): string {
  const map: Record<string, string> = {
    D1: 'subsidized', D6: 'subsidized',
    D2: 'unsubsidized', D5: 'consolidated',
    D3: 'grad_plus', D4: 'parent_plus',
    // FFEL
    SF: 'subsidized', SU: 'unsubsidized',
    GP: 'grad_plus', PP: 'parent_plus',
    CL: 'consolidated',
    // Perkins
    PK: 'perkins',
  };
  return map[code.toUpperCase()] || 'other';
}

function titleCase(s: string): string {
  return s.toLowerCase().replace(/\b\w/g, (c) => c.toUpperCase());
}

/**
 * Parse CSV loan data. Expects header row with columns like:
 * name/type, balance, rate, servicer, status
 */
export function parseCSV(text: string): { loans: ParsedLoan[]; warnings: string[] } {
  const loans: ParsedLoan[] = [];
  const warnings: string[] = [];
  const lines = text.split('\n').filter((l) => l.trim());

  if (lines.length < 2) {
    warnings.push('CSV file appears empty or has no data rows.');
    return { loans, warnings };
  }

  // Parse header
  const delimiter = lines[0].includes('\t') ? '\t' : ',';
  const headers = lines[0].split(delimiter).map((h) => h.trim().toLowerCase().replace(/['"]/g, ''));

  // Map columns
  const colMap: Record<string, number> = {};
  const aliases: Record<string, string[]> = {
    name: ['name', 'loan', 'loan name', 'description', 'loan type', 'type'],
    balance: ['balance', 'amount', 'principal', 'outstanding', 'current balance'],
    rate: ['rate', 'interest', 'interest rate', 'apr'],
    servicer: ['servicer', 'loan servicer', 'lender', 'provider'],
    status: ['status', 'loan status', 'repayment status'],
  };

  for (const [field, names] of Object.entries(aliases)) {
    const idx = headers.findIndex((h) => names.some((n) => h.includes(n)));
    if (idx >= 0) colMap[field] = idx;
  }

  if (!('balance' in colMap)) {
    warnings.push('Could not find a "balance" column. Looking for numeric values instead.');
  }

  // Parse data rows
  for (let i = 1; i < lines.length; i++) {
    const cells = lines[i].split(delimiter).map((c) => c.trim().replace(/^["']|["']$/g, ''));
    if (cells.length < 2) continue;

    const loan: ParsedLoan = {
      name: cells[colMap.name] || `Loan ${i}`,
      loanType: detectLoanType(cells[colMap.name] || ''),
      balance: parseDollar(cells[colMap.balance] || '0'),
      interestRate: parsePercent(cells[colMap.rate] || '0'),
      servicer: cells[colMap.servicer] || '',
      status: cells[colMap.status] || '',
      disbursementDate: '',
      outstandingInterest: 0,
    };

    if (loan.balance > 0 || loan.name) {
      loans.push(loan);
    }
  }

  return { loans, warnings };
}

/**
 * Extract text from a PDF file using pdfjs-dist (client-side).
 * Returns the raw text content of all pages.
 */
export async function extractPDFText(file: File): Promise<string> {
  // Dynamic import to avoid SSR issues
  const pdfjsLib = await import('pdfjs-dist');

  // Set worker source
  pdfjsLib.GlobalWorkerOptions.workerSrc = `https://cdnjs.cloudflare.com/ajax/libs/pdf.js/${pdfjsLib.version}/pdf.worker.min.mjs`;

  const arrayBuffer = await file.arrayBuffer();
  const pdf = await pdfjsLib.getDocument({ data: arrayBuffer }).promise;

  const pages: string[] = [];
  for (let i = 1; i <= pdf.numPages; i++) {
    const page = await pdf.getPage(i);
    const content = await page.getTextContent();
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    const text = content.items
      .map((item: any) => item.str || '')
      .join(' ');
    pages.push(text);
  }

  return pages.join('\n\n');
}

/**
 * Parse a servicer statement (typically PDF text).
 * Extracts balance, payment due, and servicer name.
 */
export function parseServicerStatement(text: string): { loans: ParsedLoan[]; warnings: string[] } {
  const loans: ParsedLoan[] = [];
  const warnings: string[] = [];

  // Try to find servicer name
  const servicerMatch = text.match(/(?:MOHELA|Nelnet|Aidvantage|Edfinancial|Great\s*Lakes|FedLoan|ECSI|Heartland)/i);
  const servicer = servicerMatch ? servicerMatch[0] : '';

  // Find balance(s)
  const balanceMatches = text.matchAll(/(?:(?:current|outstanding|principal)\s*balance|balance\s*(?:due|owed)?)[:\s]*\$?([\d,]+\.?\d*)/gi);
  for (const match of balanceMatches) {
    const balance = parseDollar(match[1]);
    if (balance > 100) { // Filter out small noise values
      loans.push({
        name: `Loan from ${servicer || 'Statement'}`,
        loanType: 'unknown',
        balance,
        interestRate: 0,
        servicer,
        disbursementDate: '',
        status: 'In Repayment',
        outstandingInterest: 0,
      });
    }
  }

  // Try to find interest rate
  const rateMatch = text.match(/(?:interest rate|rate)[:\s]*([\d.]+)\s*%/i);
  if (rateMatch && loans.length > 0) {
    loans[0].interestRate = parseFloat(rateMatch[1]);
  }

  // Find payment amount
  const paymentMatch = text.match(/(?:payment\s*(?:due|amount)|minimum\s*payment|monthly\s*payment)[:\s]*\$?([\d,]+\.?\d*)/i);
  if (paymentMatch) {
    warnings.push(`Monthly payment found: $${parseDollar(paymentMatch[1]).toLocaleString()}`);
  }

  if (loans.length === 0) {
    warnings.push('Could not extract loan data from this statement. You may need to enter values manually.');
  }

  return { loans, warnings };
}

/**
 * Parse ECF / PSLF tracking result.
 * Looks for qualifying payment counts.
 */
export function parseECFDocument(text: string): { pslfPayments: number; warnings: string[] } {
  const warnings: string[] = [];

  // Look for qualifying payment count
  const paymentPatterns = [
    /(\d+)\s*(?:qualifying|eligible)\s*payments?/i,
    /qualifying\s*payments?[:\s]*(\d+)/i,
    /payment\s*count[:\s]*(\d+)/i,
    /(\d+)\s*of\s*120/i,
  ];

  for (const pattern of paymentPatterns) {
    const match = text.match(pattern);
    if (match) {
      const count = parseInt(match[1], 10);
      if (count >= 0 && count <= 120) {
        return { pslfPayments: count, warnings };
      }
    }
  }

  warnings.push('Could not find qualifying payment count. Enter it manually on the PSLF Tracker page.');
  return { pslfPayments: 0, warnings };
}

/**
 * Master parse function — classifies the file and routes to the appropriate parser.
 */
export async function parseFile(file: File): Promise<ParseResult> {
  let rawText = '';
  const warnings: string[] = [];

  // Step 1: Extract text based on file type
  const ext = file.name.split('.').pop()?.toLowerCase() || '';
  const mime = file.type;

  if (ext === 'pdf' || mime === 'application/pdf') {
    try {
      rawText = await extractPDFText(file);
    } catch (err) {
      warnings.push(`PDF extraction failed: ${err instanceof Error ? err.message : 'Unknown error'}. Try a TXT or CSV file instead.`);
      return {
        classification: { type: 'unknown', confidence: 0, label: 'Unreadable PDF' },
        loans: [],
        rawText: '',
        warnings,
      };
    }
  } else {
    // Text-based files (TXT, CSV, TSV)
    rawText = await file.text();
  }

  // Step 2: Classify
  const classification = classifyDocument(rawText);

  // Step 3: Route to appropriate parser
  let loans: ParsedLoan[] = [];
  let pslfPayments: number | undefined;

  switch (classification.type) {
    case 'studentaid_txt': {
      const result = parseStudentAidTXT(rawText);
      loans = result.loans;
      warnings.push(...result.warnings);
      break;
    }
    case 'csv_loans': {
      const result = parseCSV(rawText);
      loans = result.loans;
      warnings.push(...result.warnings);
      break;
    }
    case 'ecf_pdf': {
      const result = parseECFDocument(rawText);
      pslfPayments = result.pslfPayments;
      warnings.push(...result.warnings);
      break;
    }
    case 'servicer_statement': {
      const result = parseServicerStatement(rawText);
      loans = result.loans;
      warnings.push(...result.warnings);
      break;
    }
    case '1098e': {
      warnings.push('1098-E detected. This shows interest paid for tax purposes, not loan details. Use your servicer statement or StudentAid export for loan data.');
      break;
    }
    default: {
      // Try all parsers and pick the one that extracts the most
      const txtResult = parseStudentAidTXT(rawText);
      const csvResult = parseCSV(rawText);
      const stmtResult = parseServicerStatement(rawText);

      if (txtResult.loans.length >= csvResult.loans.length && txtResult.loans.length >= stmtResult.loans.length) {
        loans = txtResult.loans;
        warnings.push(...txtResult.warnings);
      } else if (csvResult.loans.length >= stmtResult.loans.length) {
        loans = csvResult.loans;
        warnings.push(...csvResult.warnings);
      } else {
        loans = stmtResult.loans;
        warnings.push(...stmtResult.warnings);
      }

      if (loans.length === 0) {
        warnings.push('Could not auto-detect document type or extract loan data. You can enter your loans manually.');
      }
    }
  }

  return { classification, loans, pslfPayments, rawText, warnings };
}

// ── Helpers ────────────────────────────────────────────────────────

function parseDollar(s: string): number {
  return parseFloat(s.replace(/[$,\s]/g, '')) || 0;
}

function parsePercent(s: string): number {
  const n = parseFloat(s.replace(/[%\s]/g, ''));
  return isNaN(n) ? 0 : n;
}

function detectLoanType(text: string): string {
  const t = text.toLowerCase();
  if (t.includes('subsidized') && !t.includes('unsubsidized')) return 'subsidized';
  if (t.includes('unsubsidized') || t.includes('unsub')) return 'unsubsidized';
  if (t.includes('grad plus') || t.includes('grad-plus') || t.includes('graduate plus')) return 'grad_plus';
  if (t.includes('parent plus') || t.includes('parent-plus')) return 'parent_plus';
  if (t.includes('consolidat')) return 'consolidated';
  if (t.includes('perkins')) return 'perkins';
  return 'other';
}

function finalizeLoan(partial: Partial<ParsedLoan>): ParsedLoan {
  return {
    name: partial.name || 'Unknown Loan',
    loanType: partial.loanType || 'other',
    balance: partial.balance || 0,
    interestRate: partial.interestRate || 0,
    servicer: partial.servicer || '',
    disbursementDate: partial.disbursementDate || '',
    status: partial.status || '',
    outstandingInterest: partial.outstandingInterest || 0,
  };
}