pipeline: DFPI escrow+CFL scraper (3532 rows) + attorney reuse-pull into realestate DB

ba624d2587eb098bca1119818b8ed6ca8801129c · 2026-07-30 21:31:29 -0700 · Steve
Files touched

A scrapers/dfpi.mjs
Diff

commit ba624d2587eb098bca1119818b8ed6ca8801129c
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Jul 30 21:31:29 2026 -0700

    pipeline: DFPI escrow+CFL scraper (3532 rows) + attorney reuse-pull into realestate DB
---
 scrapers/dfpi.mjs | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)

diff --git a/scrapers/dfpi.mjs b/scrapers/dfpi.mjs
new file mode 100644
index 0000000..99d2ffc
--- /dev/null
+++ b/scrapers/dfpi.mjs
@@ -0,0 +1,66 @@
+#!/usr/bin/env node
+// DFPI (CA) Solr feed → CSV for realestate.rentv_licensed_targets.
+// Escrow + CFL (commercial-lender proxy), Active + SoCal ZIP only. Feed-first, $0.
+import { writeFileSync } from 'node:fs';
+
+const ENDPOINT = 'https://searchcloud-1-us-west-2.searchstax.com/29847/dfpiprod-1839/emselect';
+const TOKEN = 'cd1f7b503538c28008a908b324dcc2e8c60a4a3e'; // public select-token (re-pull from page if 401)
+const HEADERS = { Authorization: 'Token ' + TOKEN, Origin: 'https://dfpi.ca.gov', Referer: 'https://dfpi.ca.gov/' };
+
+const SOURCES = [
+  { source: 'dfpi_escrow', role: 'Escrow',  industry: 'Escrow' },
+  { source: 'dfpi_cfl',    role: 'Lender',  industry: 'California Finance Lender and Broker' },
+];
+
+function market(zip) {
+  const z = parseInt(zip, 10); if (!z) return null;
+  if (z >= 90001 && z <= 91899) return 'Greater LA';
+  if (z >= 91900 && z <= 92199) return 'San Diego';
+  if (z >= 92200 && z <= 92599) return 'Inland Empire';
+  if (z >= 92600 && z <= 92899) return 'Orange County';
+  if (z >= 93000 && z <= 93099) return 'Ventura';
+  return 'SoCal (other)';
+}
+const first = v => Array.isArray(v) ? v[0] : v;
+const clean = v => { const s = first(v); return (s == null || String(s) === '\x00') ? '' : String(s).trim(); };
+const csvCell = v => '"' + String(v == null ? '' : v).replace(/"/g, '""') + '"';
+
+async function pull(src) {
+  const fq = [
+    'fq=ss_content_type_s:%22Regulated%20Entity%22',
+    'fq=ss_industry_s:' + encodeURIComponent('"' + src.industry + '"'),
+    'fq=status_s:Active',
+    'fq=Phy_Zip:%5B90001%20TO%2093599%5D',
+  ].join('&');
+  const rows = []; let start = 0, total = Infinity;
+  while (start < total) {
+    const url = `${ENDPOINT}?q=*:*&${fq}&fl=*&wt=json&rows=500&start=${start}`;
+    const r = await fetch(url, { headers: HEADERS });
+    if (!r.ok) { console.error(src.source, 'HTTP', r.status); break; }
+    const j = await r.json();
+    total = j.response.numFound;
+    for (const d of j.response.docs) {
+      const zip = clean(d.Phy_Zip);
+      rows.push([
+        src.source, src.role,
+        clean(d.Legal_Name_Organization) || clean(d.title_t), '',      // entity, contact
+        clean(d.License_Number), clean(d.ss_industry_s), clean(d.status_s),
+        clean(d.Phy_Address1), clean(d.Phy_City), '', 'CA', zip, '', '', // address,city,county,state,zip,phone,website
+        market(zip), 't', 't',                                          // market, commercial_flag, within_300mi
+        'https://dfpi.ca.gov' + clean(d.uri),
+        JSON.stringify({ id: clean(d.id), dba: clean(d.Organization_DBA), licensed_on: clean(d.Originally_Licensed_On), enf: clean(d.ENFCASES) }),
+      ]);
+    }
+    start += 500;
+    await new Promise(res => setTimeout(res, 300)); // polite
+  }
+  console.error(src.source, 'pulled', rows.length, 'of', total, 'Active SoCal');
+  return rows;
+}
+
+const all = [];
+for (const s of SOURCES) all.push(...await pull(s));
+const cols = ['source','role','entity_name','contact_name','license_no','license_type','license_status','address','city','county','state','zip','phone','website','market','commercial_flag','within_300mi','source_url','raw'];
+const csv = all.map(r => r.map(csvCell).join(',')).join('\n') + '\n';
+writeFileSync('/tmp/dfpi_rows.csv', csv);
+console.error('wrote /tmp/dfpi_rows.csv —', all.length, 'rows; cols:', cols.join(','));
← eaa8236 specs COMPLETE: contractors (CSLB direct CSV + AZ ROC commer · back to Rentv Licensed Targets · scraper: CA DRE subdivider -> CA developers 9beee1a →