`src/ingest/brokers/il_idfpr.ts`

72 lines
/**
 * IL — IDFPR "Professional Licensing" on illinois-edp.data.socrata.com
 * (pzzh-kp68; 4.26M rows all professions, probed 2026-07-21). We pull only
 * license_type='REAL ESTATE' via SODA CSV (663,185 rows / 593,494 distinct
 * license numbers — renewal-era duplicate rows exist, so pages are ordered by
 * license_number + lastmodifieddate ascending: engine's last-write-wins keeps
 * the newest row). `description` is the specific credential (SALESPERSON /
 * BROKER / MANAGING BROKER / BROKER CORPORATION / …) and becomes our
 * license_type. Education credentials (CE/pre-license courses, schools,
 * instructors) are skipped. No employer linkage columns → individuals carry
 * no firm, like CO.
 */
import { download } from '../run.ts';
import { parseCsvObjects } from './csv.ts';
import type { BrokerRow, FetchedFile, StateAdapter } from './types.ts';

const PAGE = 300_000;
const MAX_PAGES = 10;
const BASE = 'https://illinois-edp.data.socrata.com/resource/pzzh-kp68.csv';
const QUERY = '?$where=' + encodeURIComponent("license_type='REAL ESTATE'")
  + '&$order=' + encodeURIComponent('license_number,lastmodifieddate')
  + `&$limit=${PAGE}`;

const EDUCATION = /CE (COURSE|INSTRUCTOR|SCHOOL|SPONSOR)|PRE-LICENSE|EDUCATION PROVIDER/;
const FIRM = /CORPORATION|LIMITED LIABILITY|PARTNERSHIP|ASSOCIATION|BRANCH OFFICE|VIRTUAL OFFICE/;

export const ilIdfpr: StateAdapter = {
  state: 'IL',
  source: 'il_idfpr',
  url: BASE + QUERY,
  async fetch() {
    const out: FetchedFile[] = [];
    for (let p = 0; p < MAX_PAGES; p++) {
      const f = await download(`${BASE}${QUERY}&$offset=${p * PAGE}`, `il_idfpr_p${p}.csv`);
      const { statSync } = await import('node:fs');
      out.push(f);
      // header-only page (~300 bytes) = past the last row
      if (statSync(f.path).size < 1000) break;
      await new Promise(r => setTimeout(r, 500)); // ≤2 req/s
    }
    return out;
  },
  *parse(files: FetchedFile[]): Generator<BrokerRow> {
    for (const file of files) {
      for (const o of parseCsvObjects(file.path)) {
        const desc = (o.description || '').toUpperCase();
        if (!o.license_number || !desc || EDUCATION.test(desc)) continue;
        const status = o.license_status || null;
        const city = o.city || null;
        const st = (o.state || '').toUpperCase() || null;
        const personName = [o.first_name, o.middle, o.last_name].filter(Boolean).join(' ');
        const bizName = o.business_name || o.businessdba || '';
        if (FIRM.test(desc) || o.business === 'Y') {
          const name = bizName || personName;
          if (!name) continue;
          yield {
            kind: 'firm', name, license_no: o.license_number,
            license_type: desc, license_status: status, city, state_code: st,
          };
        } else {
          const name = personName || bizName;
          if (!name) continue;
          yield {
            kind: 'broker', name, license_no: o.license_number,
            license_type: desc, license_status: status, city, state_code: st,
          };
        }
      }
    }
  },
};