← back to Nationalrealestate
Broker wave-2: DE DPR adapter ingested (29,895 brokers + 997 firms via data.delaware.gov SODA), IL IDFPR adapter added
93eff11977f50aaabfe06cf2e10cdf273020c4a6 · 2026-07-21 17:20:10 -0700 · Steve Abrams
Files touched
A src/ingest/brokers/de_dpr.tsM src/ingest/brokers/engine.tsA src/ingest/brokers/il_idfpr.ts
Diff
commit 93eff11977f50aaabfe06cf2e10cdf273020c4a6
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Jul 21 17:20:10 2026 -0700
Broker wave-2: DE DPR adapter ingested (29,895 brokers + 997 firms via data.delaware.gov SODA), IL IDFPR adapter added
---
src/ingest/brokers/de_dpr.ts | 53 +++++++++++++++++++++++++++++++
src/ingest/brokers/engine.ts | 4 ++-
src/ingest/brokers/il_idfpr.ts | 71 ++++++++++++++++++++++++++++++++++++++++++
3 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/src/ingest/brokers/de_dpr.ts b/src/ingest/brokers/de_dpr.ts
new file mode 100644
index 0000000..920d14e
--- /dev/null
+++ b/src/ingest/brokers/de_dpr.ts
@@ -0,0 +1,53 @@
+/**
+ * DE — Division of Professional Regulation "Professional and Occupational
+ * Licensing" on data.delaware.gov (pjnv-eaih; 351k rows all professions,
+ * probed 2026-07-21). We pull profession_id='Real Estate' via SODA CSV
+ * (30,895 rows): Salesperson / Resident Salesperson / Nonresident Salesperson
+ * / Broker / Resident Broker / Nonresident Broker / Associate Broker
+ * individuals, plus Main/Branch Office Permit rows which are brokerage
+ * offices → firm rows (company name arrives in last_name/combined_name).
+ * All statuses kept (Active/Expired/Closed…), license_status preserved.
+ * No employer linkage columns → individuals carry no firm.
+ */
+import { download } from '../run.ts';
+import { parseCsvObjects } from './csv.ts';
+import type { BrokerRow, FetchedFile, StateAdapter } from './types.ts';
+
+const URL = 'https://data.delaware.gov/resource/pjnv-eaih.csv'
+ + '?$where=' + encodeURIComponent("profession_id='Real Estate'")
+ + '&$order=license_no&$limit=100000';
+
+const OFFICE = /Office Permit/i;
+
+export const deDpr: StateAdapter = {
+ state: 'DE',
+ source: 'de_dpr',
+ url: URL,
+ async fetch() {
+ return [await download(URL, 'de_dpr_real_estate.csv')];
+ },
+ *parse(files: FetchedFile[]): Generator<BrokerRow> {
+ for (const o of parseCsvObjects(files[0].path)) {
+ const type = o.license_type || '';
+ if (!o.license_no || !type) continue;
+ const status = o.license_status || null;
+ const city = o.city || null;
+ const st = (o.state || '').toUpperCase() || null;
+ if (OFFICE.test(type)) {
+ const name = o.combined_name || o.last_name;
+ if (!name) continue;
+ yield {
+ kind: 'firm', name, license_no: o.license_no,
+ license_type: type, license_status: status, city, state_code: st,
+ };
+ } else {
+ const name = [o.first_name, o.last_name].filter(Boolean).join(' ') || o.combined_name;
+ if (!name) continue;
+ yield {
+ kind: 'broker', name, license_no: o.license_no,
+ license_type: type, license_status: status, city, state_code: st,
+ };
+ }
+ }
+ },
+};
diff --git a/src/ingest/brokers/engine.ts b/src/ingest/brokers/engine.ts
index 38fff8d..0921f71 100644
--- a/src/ingest/brokers/engine.ts
+++ b/src/ingest/brokers/engine.ts
@@ -17,8 +17,10 @@ import { flDbpr } from './fl_dbpr.ts';
import { nyDos } from './ny_dos.ts';
import { coDre } from './co_dre.ts';
import { ctDcp } from './ct_dcp.ts';
+import { ilIdfpr } from './il_idfpr.ts';
+import { deDpr } from './de_dpr.ts';
-const ADAPTERS: StateAdapter[] = [txTrec, flDbpr, nyDos, coDre, ctDcp];
+const ADAPTERS: StateAdapter[] = [txTrec, flDbpr, nyDos, coDre, ctDcp, ilIdfpr, deDpr];
const BATCH = 5000;
export function normalizeName(name: string): string {
diff --git a/src/ingest/brokers/il_idfpr.ts b/src/ingest/brokers/il_idfpr.ts
new file mode 100644
index 0000000..64e664c
--- /dev/null
+++ b/src/ingest/brokers/il_idfpr.ts
@@ -0,0 +1,71 @@
+/**
+ * IL — IDFPR "Professional Licensing" on illinois-edp.data.socrata.com
+ * (pzzh-kp68; 4.26M rows all professions, probed 2026-07-21). We pull only
+ * license_type='REAL ESTATE' via SODA CSV (663,185 rows / 593,494 distinct
+ * license numbers — renewal-era duplicate rows exist, so pages are ordered by
+ * license_number + lastmodifieddate ascending: engine's last-write-wins keeps
+ * the newest row). `description` is the specific credential (SALESPERSON /
+ * BROKER / MANAGING BROKER / BROKER CORPORATION / …) and becomes our
+ * license_type. Education credentials (CE/pre-license courses, schools,
+ * instructors) are skipped. No employer linkage columns → individuals carry
+ * no firm, like CO.
+ */
+import { download } from '../run.ts';
+import { parseCsvObjects } from './csv.ts';
+import type { BrokerRow, FetchedFile, StateAdapter } from './types.ts';
+
+const PAGE = 300_000;
+const MAX_PAGES = 10;
+const BASE = 'https://illinois-edp.data.socrata.com/resource/pzzh-kp68.csv';
+const QUERY = '?$where=' + encodeURIComponent("license_type='REAL ESTATE'")
+ + '&$order=' + encodeURIComponent('license_number,lastmodifieddate')
+ + `&$limit=${PAGE}`;
+
+const EDUCATION = /CE (COURSE|INSTRUCTOR|SCHOOL|SPONSOR)|PRE-LICENSE|EDUCATION PROVIDER/;
+const FIRM = /CORPORATION|LIMITED LIABILITY|PARTNERSHIP|ASSOCIATION|BRANCH OFFICE|VIRTUAL OFFICE/;
+
+export const ilIdfpr: StateAdapter = {
+ state: 'IL',
+ source: 'il_idfpr',
+ url: BASE + QUERY,
+ async fetch() {
+ const out: FetchedFile[] = [];
+ for (let p = 0; p < MAX_PAGES; p++) {
+ const f = await download(`${BASE}${QUERY}&$offset=${p * PAGE}`, `il_idfpr_p${p}.csv`);
+ const { statSync } = await import('node:fs');
+ out.push(f);
+ // header-only page (~300 bytes) = past the last row
+ if (statSync(f.path).size < 1000) break;
+ await new Promise(r => setTimeout(r, 500)); // ≤2 req/s
+ }
+ return out;
+ },
+ *parse(files: FetchedFile[]): Generator<BrokerRow> {
+ for (const file of files) {
+ for (const o of parseCsvObjects(file.path)) {
+ const desc = (o.description || '').toUpperCase();
+ if (!o.license_number || !desc || EDUCATION.test(desc)) continue;
+ const status = o.license_status || null;
+ const city = o.city || null;
+ const st = (o.state || '').toUpperCase() || null;
+ const personName = [o.first_name, o.middle, o.last_name].filter(Boolean).join(' ');
+ const bizName = o.business_name || o.businessdba || '';
+ if (FIRM.test(desc) || o.business === 'Y') {
+ const name = bizName || personName;
+ if (!name) continue;
+ yield {
+ kind: 'firm', name, license_no: o.license_number,
+ license_type: desc, license_status: status, city, state_code: st,
+ };
+ } else {
+ const name = personName || bizName;
+ if (!name) continue;
+ yield {
+ kind: 'broker', name, license_no: o.license_number,
+ license_type: desc, license_status: status, city, state_code: st,
+ };
+ }
+ }
+ }
+ },
+};
← 1c7f851 auto-save: 2026-07-21T17:08:42 (8 files) — package.json publ
·
back to Nationalrealestate
·
Consolidate agent M-P2/wave-2 work: Cook IL parcel adapter, a808341 →