`scripts/fetch-fha-condos.js`

232 lines
// fetch-fha-condos.js — ingest the PUBLIC HUD FHA-approved condominium list for Los Angeles County.
//
// Source: https://entp.hud.gov/idapp/html/condlook.cfm  (HUD's public Condominiums Search form).
// It's a ColdFusion form that POSTs to condo1.cfm. Decoded form mechanics (no Browserbase needed):
//   - landing page sets CFID/CFTOKEN cookies; we carry them.
//   - search POSTs fstate=CA + fcounty="LOS ANGELES" + fstatus_code (A|E|X) + fsearch_type=P.
//   - results are a 13-column HTML table: Condo Name, Condo ID/Submission, Address, County,
//     Composition, Manufactured Housing, FHA Concentration, Approval Method, Document Status,
//     Status, Comments, Status Date, Expiration Date.
//   - pagination is a hidden startAt/maxRows cursor; we loop startAt += pageSize until a page is empty.
//
// Output: data/fha-approved-condos.json  — every LA County project with status Approved or Expired,
// the warrantability signal for the classifier. Plain Node https, $0 (local + public gov form).
//
// Run: node scripts/fetch-fha-condos.js  [--status A|E|X]   (default: fetch A then E)

'use strict';
const https = require('https');
const fs = require('fs');
const path = require('path');
const zlib = require('zlib');

const HOST = 'entp.hud.gov';
const LANDING = '/idapp/html/condlook.cfm';
const SEARCH = '/idapp/html/condo1.cfm';
const OUT = path.join(__dirname, '..', 'data', 'fha-approved-condos.json');
const PAGE = 50;
const MAX_PAGES = 200; // hard safety cap (200 * 50 = 10k rows)

function req(method, pathname, { headers = {}, body = null } = {}) {
  return new Promise((resolve, reject) => {
    const r = https.request({
      host: HOST, path: pathname, method,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml',
        'Accept-Encoding': 'gzip, deflate',
        ...headers
      }
    }, res => {
      const chunks = [];
      res.on('data', c => chunks.push(c));
      res.on('end', () => {
        let buf = Buffer.concat(chunks);
        const enc = res.headers['content-encoding'];
        try {
          if (enc === 'gzip') buf = zlib.gunzipSync(buf);
          else if (enc === 'deflate') buf = zlib.inflateSync(buf);
        } catch (_) {}
        resolve({ status: res.statusCode, headers: res.headers, body: buf.toString('utf8') });
      });
    });
    r.on('error', reject);
    r.setTimeout(60000, () => r.destroy(new Error('timeout')));
    if (body) r.write(body);
    r.end();
  });
}

function cookiesFrom(setCookie) {
  if (!setCookie) return '';
  return setCookie.map(c => c.split(';')[0]).join('; ');
}

const clean = s => s.replace(/<[^>]+>/g, ' ').replace(/&nbsp;/gi, ' ').replace(/&amp;/gi, '&')
  .replace(/&#\d+;/g, ' ').replace(/\s+/g, ' ').trim();

// Parse the 13-column result table. Returns array of row objects. Header row is the one whose
// first cell is "Condo Name"; data rows follow until a non-13-cell row.
function parseRows(html) {
  const trs = html.match(/<tr[^>]*>[\s\S]*?<\/tr>/gi) || [];
  const out = [];
  for (const tr of trs) {
    const cells = (tr.match(/<td[^>]*>[\s\S]*?<\/td>/gi) || []).map(clean);
    if (cells.length < 13) continue;
    const name = cells[0];
    if (!name || /^condo name$/i.test(name)) continue; // skip header
    const idsub = cells[1].split(/\s+/);
    out.push({
      project_name: name.replace(/\*+$/, '').trim(),
      condo_id: idsub[0] || null,
      submission: idsub[1] || null,
      address_raw: cells[2],
      county: cells[3],
      composition: cells[4],
      manufactured_housing: cells[5],
      fha_concentration: cells[6],
      approval_method: cells[7],
      document_status: cells[8],
      status: cells[9],
      comments: cells[10],
      status_date: cells[11],
      expiration_date: cells[12]
    });
  }
  return out;
}

// LA County multi-word city names — HUD writes them without a delimiter between street and city,
// so a naive "last token is the city" split mangles them (REDONDO BEACH -> BEACH). Longest-match these.
const MULTI_WORD_CITIES = [
  'RANCHO PALOS VERDES', 'SANTA FE SPRINGS', 'LA CANADA FLINTRIDGE', 'PALOS VERDES ESTATES',
  'NORTH HOLLYWOOD', 'WEST HOLLYWOOD', 'WOODLAND HILLS', 'CANOGA PARK', 'SHERMAN OAKS',
  'STUDIO CITY', 'VALLEY VILLAGE', 'VAN NUYS', 'MISSION HILLS', 'GRANADA HILLS',
  'PORTER RANCH', 'SIMI VALLEY', 'CULVER CITY', 'MARINA DEL REY', 'PLAYA DEL REY',
  'PLAYA VISTA', 'EL SEGUNDO', 'MANHATTAN BEACH', 'HERMOSA BEACH', 'REDONDO BEACH',
  'LONG BEACH', 'SIGNAL HILL', 'SANTA MONICA', 'PACIFIC PALISADES', 'BEVERLY HILLS',
  'WEST COVINA', 'WEST HILLS', 'SANTA CLARITA', 'BALDWIN PARK', 'MONTEREY PARK',
  'TEMPLE CITY', 'SOUTH PASADENA', 'SOUTH GATE', 'SOUTH EL MONTE', 'EL MONTE',
  'SAN DIMAS', 'SAN GABRIEL', 'SAN FERNANDO', 'SAN MARINO', 'SAN PEDRO',
  'HACIENDA HEIGHTS', 'ROWLAND HEIGHTS', 'DIAMOND BAR', 'CITY OF INDUSTRY',
  'PICO RIVERA', 'SANTA FE', 'WINNETKA', 'VALENCIA', 'STEVENSON RANCH',
  'AGOURA HILLS', 'CALABASAS', 'WESTLAKE VILLAGE', 'MONROVIA', 'SIERRA MADRE',
  'LA MIRADA', 'LA PUENTE', 'LA VERNE', 'LA HABRA HEIGHTS', 'LAKEWOOD',
  'HUNTINGTON PARK', 'BELL GARDENS', 'CANYON COUNTRY', 'NEWHALL', 'SAUGUS',
  'TOLUCA LAKE', 'NORTH HILLS', 'PANORAMA CITY', 'ARLETA', 'PACOIMA',
  'MONTEBELLO', 'LOS ANGELES'
];

// Split the HUD address blob ("10925 BLIX STREET NORTH HOLLYWOOD, CA 91601") into parts.
function splitAddress(raw) {
  const m = raw.match(/^(.*?),?\s*(CA)\s+(\d{5})(?:-\d{4})?$/i);
  let street = raw, city = '', zip = '';
  if (m) {
    zip = m[3];
    const head = m[1].trim();
    const headUp = head.toUpperCase();
    // Longest known multi-word city that the head ends with wins.
    const hit = MULTI_WORD_CITIES.filter(c => headUp.endsWith(' ' + c) || headUp === c)
      .sort((a, b) => b.length - a.length)[0];
    if (hit) {
      city = hit;
      street = head.slice(0, head.length - hit.length).trim();
    } else {
      const cm = head.match(/^(.*\S)\s+([A-Z][A-Z .'-]+)$/); // fallback: trailing single-word CITY
      if (cm) { street = cm[1].trim(); city = cm[2].trim(); }
      else { street = head; }
    }
  }
  return { street, city, zip };
}

// The initial search MUST NOT send startAt/maxRows (sending startAt=0 yields a 500). Pagination is
// driven entirely by the hidden next-page form HUD renders, which carries the live startAt/maxRows.
function nextCursor(html) {
  // Find a form (any case) whose inputs include a startAt — that's the "next" form.
  const forms = html.match(/<form[\s\S]*?<\/form>/gi) || [];
  for (const f of forms) {
    const sa = f.match(/name="?startAt"?\s+value="?(\d+)"?/i);
    if (!sa) continue;
    const mr = f.match(/name="?maxRows"?\s+value="?(\d+)"?/i);
    return { startAt: sa[1], maxRows: (mr && mr[1]) || String(PAGE) };
  }
  return null;
}

async function fetchStatus(statusCode, cookie) {
  const all = [];
  let cursor = null;
  for (let page = 0; page < MAX_PAGES; page++) {
    const fields = {
      fapproval_method: 'NEW', fsorted_by: 'condo_name', fstate: 'CA',
      fcounty: 'LOS ANGELES', fcondo_id: '', fcondo_name: '', fcity: '', fzip: '',
      fstatus_code: statusCode, fsearch_type: 'P',
      fbegin_mo: '', fbegin_dy: '', fbegin_yr: '', fend_mo: '', fend_dy: '', fend_yr: '',
      came_from: 'oth', in_fhac: 'true'
    };
    if (cursor) { fields.startAt = cursor.startAt; fields.maxRows = cursor.maxRows; fields.thisPage = 'condo1.cfm'; }
    const body = new URLSearchParams(fields).toString();
    const res = await req('POST', SEARCH, {
      headers: {
        'Content-Type': 'application/x-www-form-urlencoded',
        'Content-Length': Buffer.byteLength(body),
        'Cookie': cookie,
        'Referer': `https://${HOST}${LANDING}`
      },
      body
    });
    const rows = parseRows(res.body);
    if (!rows.length) break;
    all.push(...rows);
    process.stderr.write(`  status=${statusCode} page ${page} (startAt=${cursor ? cursor.startAt : 0}): +${rows.length} (total ${all.length})\n`);
    cursor = nextCursor(res.body);
    if (!cursor) break; // no more pages
    await new Promise(r => setTimeout(r, 400)); // be polite to a gov server
  }
  return all;
}

async function main() {
  const argStatus = (process.argv.find(a => a.startsWith('--status=')) || '').split('=')[1];
  const statuses = argStatus ? [argStatus] : ['A', 'E']; // Approved + Expired

  process.stderr.write('Fetching HUD condo-lookup landing (for CFID/CFTOKEN cookies)...\n');
  const landing = await req('GET', LANDING);
  const cookie = cookiesFrom(landing.headers['set-cookie']);
  if (!cookie) throw new Error('no session cookie from HUD landing page');

  const byKey = new Map();
  for (const s of statuses) {
    const rows = await fetchStatus(s, cookie);
    for (const r of rows) {
      const { street, city, zip } = splitAddress(r.address_raw);
      const rec = { ...r, address: street, city, zip,
        warrant_signal: /approved/i.test(r.status) ? 'fha_approved'
          : /expired/i.test(r.status) ? 'fha_expired' : 'fha_other' };
      const key = (r.condo_id || '') + '|' + (r.submission || '') + '|' + r.project_name;
      byKey.set(key, rec); // last write wins; Approved fetched before Expired so Approved keeps
    }
  }

  const records = [...byKey.values()];
  const approved = records.filter(r => r.warrant_signal === 'fha_approved').length;
  const expired = records.filter(r => r.warrant_signal === 'fha_expired').length;
  const payload = {
    meta: {
      source: 'HUD FHA-approved condominium list (entp.hud.gov/idapp/html/condlook.cfm)',
      authority: 'U.S. Department of Housing and Urban Development — public FHA condo approval registry',
      county: 'Los Angeles County, CA',
      statuses_fetched: statuses,
      fetched_at: new Date().toISOString(),
      counts: { total: records.length, approved, expired },
      label: 'FHA-approval-based warrantability PROXY — on-list = FHA financing-eligible (not lender-verified Fannie/Freddie warrantability).'
    },
    condos: records
  };
  fs.writeFileSync(OUT, JSON.stringify(payload, null, 2));
  process.stderr.write(`\nWrote ${records.length} LA County FHA condo records (${approved} approved, ${expired} expired) -> ${OUT}\n`);
}

main().catch(e => { process.stderr.write('FATAL: ' + (e.stack || e.message) + '\n'); process.exit(1); });