`scrapers/dfpi.mjs`

67 lines
#!/usr/bin/env node
// DFPI (CA) Solr feed → CSV for realestate.rentv_licensed_targets.
// Escrow + CFL (commercial-lender proxy), Active + SoCal ZIP only. Feed-first, $0.
import { writeFileSync } from 'node:fs';

const ENDPOINT = 'https://searchcloud-1-us-west-2.searchstax.com/29847/dfpiprod-1839/emselect';
const TOKEN = 'cd1f7b503538c28008a908b324dcc2e8c60a4a3e'; // public select-token (re-pull from page if 401)
const HEADERS = { Authorization: 'Token ' + TOKEN, Origin: 'https://dfpi.ca.gov', Referer: 'https://dfpi.ca.gov/' };

const SOURCES = [
  { source: 'dfpi_escrow', role: 'Escrow',  industry: 'Escrow' },
  { source: 'dfpi_cfl',    role: 'Lender',  industry: 'California Finance Lender and Broker' },
];

function market(zip) {
  const z = parseInt(zip, 10); if (!z) return null;
  if (z >= 90001 && z <= 91899) return 'Greater LA';
  if (z >= 91900 && z <= 92199) return 'San Diego';
  if (z >= 92200 && z <= 92599) return 'Inland Empire';
  if (z >= 92600 && z <= 92899) return 'Orange County';
  if (z >= 93000 && z <= 93099) return 'Ventura';
  return 'SoCal (other)';
}
const first = v => Array.isArray(v) ? v[0] : v;
const clean = v => { const s = first(v); return (s == null || String(s) === '\x00') ? '' : String(s).trim(); };
const csvCell = v => '"' + String(v == null ? '' : v).replace(/"/g, '""') + '"';

async function pull(src) {
  const fq = [
    'fq=ss_content_type_s:%22Regulated%20Entity%22',
    'fq=ss_industry_s:' + encodeURIComponent('"' + src.industry + '"'),
    'fq=status_s:Active',
    'fq=Phy_Zip:%5B90001%20TO%2093599%5D',
  ].join('&');
  const rows = []; let start = 0, total = Infinity;
  while (start < total) {
    const url = `${ENDPOINT}?q=*:*&${fq}&fl=*&wt=json&rows=500&start=${start}`;
    const r = await fetch(url, { headers: HEADERS });
    if (!r.ok) { console.error(src.source, 'HTTP', r.status); break; }
    const j = await r.json();
    total = j.response.numFound;
    for (const d of j.response.docs) {
      const zip = clean(d.Phy_Zip);
      rows.push([
        src.source, src.role,
        clean(d.Legal_Name_Organization) || clean(d.title_t), '',      // entity, contact
        clean(d.License_Number), clean(d.ss_industry_s), clean(d.status_s),
        clean(d.Phy_Address1), clean(d.Phy_City), '', 'CA', zip, '', '', // address,city,county,state,zip,phone,website
        market(zip), 't', 't',                                          // market, commercial_flag, within_300mi
        'https://dfpi.ca.gov' + clean(d.uri),
        JSON.stringify({ id: clean(d.id), dba: clean(d.Organization_DBA), licensed_on: clean(d.Originally_Licensed_On), enf: clean(d.ENFCASES) }),
      ]);
    }
    start += 500;
    await new Promise(res => setTimeout(res, 300)); // polite
  }
  console.error(src.source, 'pulled', rows.length, 'of', total, 'Active SoCal');
  return rows;
}

const all = [];
for (const s of SOURCES) all.push(...await pull(s));
const cols = ['source','role','entity_name','contact_name','license_no','license_type','license_status','address','city','county','state','zip','phone','website','market','commercial_flag','within_300mi','source_url','raw'];
const csv = all.map(r => r.map(csvCell).join(',')).join('\n') + '\n';
writeFileSync('/tmp/dfpi_rows.csv', csv);
console.error('wrote /tmp/dfpi_rows.csv —', all.length, 'rows; cols:', cols.join(','));