`scrapers/az_state_bar.mjs`

203 lines
#!/usr/bin/env node
/**
 * az_state_bar.mjs — $0 local-Playwright scraper for Arizona real-estate attorneys
 * from the State Bar of Arizona public Member Directory.
 *
 * Source (discovered 2026-08-06):
 *   Public directory UI:
 *     https://azbar.org/for-legal-professionals/practice-tools-management/member-directory/
 *   Backing API (public, no login):
 *     POST https://api-proxy.azbar.org/MemberSearch/Search/?PageSize=N&Page=P
 *     JSON body carries the source-side filters:
 *       Specialization:"RE"  = Real Estate Law
 *       County:"07"          = Maricopa (Phoenix/Scottsdale metro)
 *     The endpoint needs a PUBLIC tools token (userid/password) that the directory
 *     page ships in its own JS — this scraper extracts it live at runtime (see main())
 *     rather than hardcoding it, so no secret-shaped string is committed.
 *
 * Filters AT THE SOURCE — pulls only the RE + Maricopa filtered set, never all 20k+ AZ attorneys.
 * Polite: ~1 req/s, headless, one session.
 *
 * HONESTY: writes ONLY fields the directory actually returns. No fabricated
 * bar numbers, emails, phones, firms, or websites. Empty stays empty.
 *
 * Usage:
 *   node scrapers/az_state_bar.mjs                       # RE + Maricopa (default), -> data/az_state_bar.csv
 *   node scrapers/az_state_bar.mjs --spec RE --county 07 # explicit
 *   node scrapers/az_state_bar.mjs --out data/foo.csv
 *
 * Reusable: change --spec / --county to retarget any AZ Bar practice area / county.
 * Practice-area codes: AL BY CD CR ET FL PI RE TX WC
 * County codes: Apache 01, Cochise 02, Coconino 03, Gila 04, Graham 05, Greenlee 06,
 *   La Paz 15, Maricopa 07, Mohave 08, Navajo 09, Out of State 00, Pima 10, Pinal 11,
 *   Santa Cruz 12, Yavapai 13, Yuma 14
 */
import { chromium } from 'playwright';
import { writeFileSync, mkdirSync } from 'fs';
import { dirname, resolve } from 'path';
import { fileURLToPath } from 'url';

const __dirname = dirname(fileURLToPath(import.meta.url));
const PROJ = resolve(__dirname, '..');

// ---- args ----
const argv = process.argv.slice(2);
const getArg = (k, d) => { const i = argv.indexOf('--' + k); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
const SPEC = getArg('spec', 'RE');       // Real Estate Law
const COUNTY = getArg('county', '07');   // Maricopa
const OUT = resolve(PROJ, getArg('out', 'data/az_state_bar.csv'));
const PAGE_SIZE = parseInt(getArg('pagesize', '25'), 10);
const POLITE_MS = parseInt(getArg('delay', '1100'), 10); // ~1 req/s

const DIR_URL = 'https://azbar.org/for-legal-professionals/practice-tools-management/member-directory/';
const API = 'https://api-proxy.azbar.org/MemberSearch/Search/';
// userid/password are a PUBLIC tools token baked into the member-directory page for
// its own search widget — extracted live from the page at runtime (see main()) so no
// secret-shaped string is committed to git and the scraper survives token rotation.
const HEADERS = { 'Content-Type': 'application/json; charset=UTF-8' };

const COUNTY_NAME = {
  '01': 'Apache', '02': 'Cochise', '03': 'Coconino', '04': 'Gila', '05': 'Graham',
  '06': 'Greenlee', '15': 'La Paz', '07': 'Maricopa', '08': 'Mohave', '09': 'Navajo',
  '00': 'Out of State', '10': 'Pima', '11': 'Pinal', '12': 'Santa Cruz', '13': 'Yavapai', '14': 'Yuma',
};

const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

// CSV columns match DB table rentv_licensed_targets
const COLS = ['source', 'role', 'entity_name', 'contact_name', 'license_no', 'license_type',
  'license_status', 'address', 'city', 'county', 'state', 'zip', 'phone', 'website',
  'market', 'commercial_flag', 'within_300mi', 'source_url'];

function csvCell(v) {
  if (v === null || v === undefined) return '';
  const s = String(v);
  return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
}

function mapRow(m) {
  const a = m.Address || {};
  const contact = [m.FirstName, m.MiddleName, m.LastName].filter(Boolean).join(' ').replace(/\s+/g, ' ').trim();
  const firm = (m.Company || '').trim();
  // phone: directory returns PhoneNumbers[] (often empty) + PrimaryPhone (often empty). Only use what's real.
  let phone = (m.PrimaryPhone || '').trim();
  if (!phone && Array.isArray(m.PhoneNumbers) && m.PhoneNumbers.length) {
    const first = m.PhoneNumbers[0];
    phone = (typeof first === 'string' ? first : (first?.Number || first?.PhoneNumber || '')).trim();
  }
  const addr = [a.Address1, a.Address2].filter(Boolean).join(' ').replace(/\s+/g, ' ').trim();
  return {
    source: 'az_state_bar',
    role: 'Attorney',
    entity_name: firm || contact,          // firm, or same as contact if solo
    contact_name: contact,
    license_no: (m.BarNumber || '').trim(), // AZ bar number
    license_type: 'AZ Bar',
    license_status: (m.MemberStatus || '').trim(),
    address: addr,
    city: (a.City || '').trim(),
    county: (a.County || COUNTY_NAME[COUNTY] || '').trim(),
    state: 'AZ',
    zip: (a.Zip || '').trim(),
    phone,
    website: '',                            // directory does not expose a website field
    market: 'Arizona',
    commercial_flag: 'true',                // real-estate practice = CRE-relevant
    within_300mi: 'true',                   // Maricopa/Scottsdale within 300mi-of-Scottsdale scope
    source_url: DIR_URL,
  };
}

async function main() {
  const specLabel = SPEC === 'RE' ? 'Real Estate Law' : SPEC;
  console.log(`[az_state_bar] spec=${SPEC} (${specLabel}) county=${COUNTY} (${COUNTY_NAME[COUNTY] || '?'}) pageSize=${PAGE_SIZE}`);

  const browser = await chromium.launch({ headless: true });
  const ctx = await browser.newContext();
  const page = await ctx.newPage();
  page.setDefaultTimeout(60000);

  // Establish origin/referer by loading the real directory page (also proves basic search isn't login-walled).
  await page.goto(DIR_URL, { waitUntil: 'domcontentloaded' });
  await sleep(1500);

  // Extract the PUBLIC search-widget auth (userid/password) live from the page — a
  // public token the page ships for its own directory search; runtime-fetching it
  // keeps the secret out of git and survives rotation.
  const html = await page.content();
  // The widget sets them via: setRequestHeader("userid","publictools"); setRequestHeader("password","<token>")
  const uid = (html.match(/setRequestHeader\(\s*["']userid["']\s*,\s*["']([^"']+)["']/i) || [])[1];
  const pw = (html.match(/setRequestHeader\(\s*["']password["']\s*,\s*["']([^"']+)["']/i) || [])[1];
  if (uid) HEADERS.userid = uid;
  if (pw) HEADERS.password = pw;
  if (!HEADERS.userid || !HEADERS.password) {
    console.error('[az_state_bar] FATAL: could not extract public search auth from the directory page — the widget markup likely changed');
    process.exit(1);
  }

  const bodyBase = {
    Type: 'Member', Firm: '', FuzzySearch: false, IncludeDeceased: false,
    FirstName: '', LastName: '', City: '', State: '', Zip: '',
    County: COUNTY, LanguageCode: '', Section: '', LegalNeed: '',
    Specialization: SPEC, JurisdictionCode: '', LawSchool: '',
  };

  const seen = new Set(); // dedupe by EntityNumber
  const rows = [];
  let total = null;
  let pageNo = 1;

  while (true) {
    const url = `${API}?PageSize=${PAGE_SIZE}&Page=${pageNo}&Shuffle=false&Seed=null`;
    const res = await page.evaluate(async ({ url, headers, body }) => {
      const r = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body) });
      let text = '';
      try { text = await r.text(); } catch { /* */ }
      return { status: r.status, text };
    }, { url, headers: HEADERS, body: bodyBase });

    if (res.status !== 200) {
      console.error(`[az_state_bar] page ${pageNo} HTTP ${res.status} — stopping. Body: ${res.text?.slice(0, 300)}`);
      break;
    }
    let json;
    try { json = JSON.parse(res.text); } catch (e) {
      console.error(`[az_state_bar] page ${pageNo} JSON parse fail — stopping.`); break;
    }
    if (!json.IsSuccess || !json.Result) {
      console.error(`[az_state_bar] page ${pageNo} not success (${json.Message || json.Error}) — stopping.`); break;
    }
    if (total === null) { total = json.Result.TotalCount; console.log(`[az_state_bar] TotalCount = ${total}`); }

    const batch = json.Result.Results || [];
    if (!batch.length) break;

    for (const m of batch) {
      const id = m.EntityNumber ?? `${m.BarNumber}-${m.LastName}`;
      if (seen.has(id)) continue;
      seen.add(id);
      rows.push(mapRow(m));
    }
    console.log(`[az_state_bar] page ${pageNo}: +${batch.length} (running ${rows.length}/${total})`);

    if (rows.length >= (total ?? 0) || batch.length < PAGE_SIZE) break;
    pageNo += 1;
    await sleep(POLITE_MS); // polite ~1 req/s
  }

  await browser.close();

  // write CSV
  mkdirSync(dirname(OUT), { recursive: true });
  const lines = [COLS.join(',')];
  for (const r of rows) lines.push(COLS.map((c) => csvCell(r[c])).join(','));
  writeFileSync(OUT, lines.join('\n') + '\n');

  console.log(`[az_state_bar] wrote ${rows.length} rows -> ${OUT}`);
  if (total !== null && rows.length !== total) {
    console.warn(`[az_state_bar] NOTE: rows (${rows.length}) != TotalCount (${total}).`);
  }
}

main().catch((e) => { console.error('[az_state_bar] FATAL', e); process.exit(1); });