`src/ingest/brokers/csv.ts`

52 lines
/**
 * Minimal RFC-4180 CSV parser (quoted fields, embedded commas/newlines/"" escapes).
 * Whole-file in memory — largest M-B1 file is ~65MB, fine on a 32GB workstation.
 */
import { readFileSync } from 'node:fs';

export function parseCsv(text: string): string[][] {
  const rows: string[][] = [];
  let row: string[] = [];
  let field = '';
  let inQ = false;
  for (let i = 0; i < text.length; i++) {
    const c = text[i];
    if (inQ) {
      if (c === '"') {
        if (text[i + 1] === '"') { field += '"'; i++; }
        else inQ = false;
      } else field += c;
    } else if (c === '"') inQ = true;
    else if (c === ',') { row.push(field); field = ''; }
    else if (c === '\n') { row.push(field); field = ''; rows.push(row); row = []; }
    else if (c !== '\r') field += c;
  }
  if (field !== '' || row.length) { row.push(field); rows.push(row); }
  return rows;
}

export function parseCsvFile(path: string, encoding: BufferEncoding = 'utf8'): string[][] {
  return parseCsv(readFileSync(path, { encoding }));
}

/** Header display names -> snake_case keys ("License Type" -> license_type). */
export function headerKeys(header: string[]): string[] {
  return header.map(h => h.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, ''));
}

/** Parse a headered CSV file into objects keyed by snake_cased header names. */
export function parseCsvObjects(path: string, encoding: BufferEncoding = 'utf8'): Record<string, string>[] {
  const rows = parseCsvFile(path, encoding);
  if (!rows.length) return [];
  const keys = headerKeys(rows[0]);
  const out: Record<string, string>[] = [];
  for (let i = 1; i < rows.length; i++) {
    const r = rows[i];
    if (r.length === 1 && r[0] === '') continue;
    const o: Record<string, string> = {};
    for (let j = 0; j < keys.length; j++) o[keys[j]] = (r[j] ?? '').trim();
    out.push(o);
  }
  return out;
}