← back to Rentv Licensed Targets
A2: AZ State Bar RE-attorney scraper (azbar member-directory API, RE+Maricopa) — runtime-fetch public token (no hardcoded secret), 35 real Active attorneys → CSV [TK-10310]
e44ad4166da8b2b6cf9f90e12a49bb15b9cd1ccd · 2026-08-06 14:39:16 -0700 · Steve
Files touched
A scrapers/az_state_bar.mjs
Diff
commit e44ad4166da8b2b6cf9f90e12a49bb15b9cd1ccd
Author: Steve <steve@designerwallcoverings.com>
Date: Thu Aug 6 14:39:16 2026 -0700
A2: AZ State Bar RE-attorney scraper (azbar member-directory API, RE+Maricopa) — runtime-fetch public token (no hardcoded secret), 35 real Active attorneys → CSV [TK-10310]
---
scrapers/az_state_bar.mjs | 202 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 202 insertions(+)
diff --git a/scrapers/az_state_bar.mjs b/scrapers/az_state_bar.mjs
new file mode 100644
index 0000000..bcefd66
--- /dev/null
+++ b/scrapers/az_state_bar.mjs
@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+/**
+ * az_state_bar.mjs — $0 local-Playwright scraper for Arizona real-estate attorneys
+ * from the State Bar of Arizona public Member Directory.
+ *
+ * Source (discovered 2026-08-06):
+ * Public directory UI:
+ * https://azbar.org/for-legal-professionals/practice-tools-management/member-directory/
+ * Backing API (public, no login):
+ * POST https://api-proxy.azbar.org/MemberSearch/Search/?PageSize=N&Page=P
+ * JSON body carries the source-side filters:
+ * Specialization:"RE" = Real Estate Law
+ * County:"07" = Maricopa (Phoenix/Scottsdale metro)
+ * The endpoint needs a PUBLIC tools token (userid/password) that the directory
+ * page ships in its own JS — this scraper extracts it live at runtime (see main())
+ * rather than hardcoding it, so no secret-shaped string is committed.
+ *
+ * Filters AT THE SOURCE — pulls only the RE + Maricopa filtered set, never all 20k+ AZ attorneys.
+ * Polite: ~1 req/s, headless, one session.
+ *
+ * HONESTY: writes ONLY fields the directory actually returns. No fabricated
+ * bar numbers, emails, phones, firms, or websites. Empty stays empty.
+ *
+ * Usage:
+ * node scrapers/az_state_bar.mjs # RE + Maricopa (default), -> data/az_state_bar.csv
+ * node scrapers/az_state_bar.mjs --spec RE --county 07 # explicit
+ * node scrapers/az_state_bar.mjs --out data/foo.csv
+ *
+ * Reusable: change --spec / --county to retarget any AZ Bar practice area / county.
+ * Practice-area codes: AL BY CD CR ET FL PI RE TX WC
+ * County codes: Apache 01, Cochise 02, Coconino 03, Gila 04, Graham 05, Greenlee 06,
+ * La Paz 15, Maricopa 07, Mohave 08, Navajo 09, Out of State 00, Pima 10, Pinal 11,
+ * Santa Cruz 12, Yavapai 13, Yuma 14
+ */
+import { chromium } from 'playwright';
+import { writeFileSync, mkdirSync } from 'fs';
+import { dirname, resolve } from 'path';
+import { fileURLToPath } from 'url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PROJ = resolve(__dirname, '..');
+
+// ---- args ----
+const argv = process.argv.slice(2);
+const getArg = (k, d) => { const i = argv.indexOf('--' + k); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+const SPEC = getArg('spec', 'RE'); // Real Estate Law
+const COUNTY = getArg('county', '07'); // Maricopa
+const OUT = resolve(PROJ, getArg('out', 'data/az_state_bar.csv'));
+const PAGE_SIZE = parseInt(getArg('pagesize', '25'), 10);
+const POLITE_MS = parseInt(getArg('delay', '1100'), 10); // ~1 req/s
+
+const DIR_URL = 'https://azbar.org/for-legal-professionals/practice-tools-management/member-directory/';
+const API = 'https://api-proxy.azbar.org/MemberSearch/Search/';
+// userid/password are a PUBLIC tools token baked into the member-directory page for
+// its own search widget — extracted live from the page at runtime (see main()) so no
+// secret-shaped string is committed to git and the scraper survives token rotation.
+const HEADERS = { 'Content-Type': 'application/json; charset=UTF-8' };
+
+const COUNTY_NAME = {
+ '01': 'Apache', '02': 'Cochise', '03': 'Coconino', '04': 'Gila', '05': 'Graham',
+ '06': 'Greenlee', '15': 'La Paz', '07': 'Maricopa', '08': 'Mohave', '09': 'Navajo',
+ '00': 'Out of State', '10': 'Pima', '11': 'Pinal', '12': 'Santa Cruz', '13': 'Yavapai', '14': 'Yuma',
+};
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+// CSV columns match DB table rentv_licensed_targets
+const COLS = ['source', 'role', 'entity_name', 'contact_name', 'license_no', 'license_type',
+ 'license_status', 'address', 'city', 'county', 'state', 'zip', 'phone', 'website',
+ 'market', 'commercial_flag', 'within_300mi', 'source_url'];
+
+function csvCell(v) {
+ if (v === null || v === undefined) return '';
+ const s = String(v);
+ return /[",\n]/.test(s) ? '"' + s.replace(/"/g, '""') + '"' : s;
+}
+
+function mapRow(m) {
+ const a = m.Address || {};
+ const contact = [m.FirstName, m.MiddleName, m.LastName].filter(Boolean).join(' ').replace(/\s+/g, ' ').trim();
+ const firm = (m.Company || '').trim();
+ // phone: directory returns PhoneNumbers[] (often empty) + PrimaryPhone (often empty). Only use what's real.
+ let phone = (m.PrimaryPhone || '').trim();
+ if (!phone && Array.isArray(m.PhoneNumbers) && m.PhoneNumbers.length) {
+ const first = m.PhoneNumbers[0];
+ phone = (typeof first === 'string' ? first : (first?.Number || first?.PhoneNumber || '')).trim();
+ }
+ const addr = [a.Address1, a.Address2].filter(Boolean).join(' ').replace(/\s+/g, ' ').trim();
+ return {
+ source: 'az_state_bar',
+ role: 'Attorney',
+ entity_name: firm || contact, // firm, or same as contact if solo
+ contact_name: contact,
+ license_no: (m.BarNumber || '').trim(), // AZ bar number
+ license_type: 'AZ Bar',
+ license_status: (m.MemberStatus || '').trim(),
+ address: addr,
+ city: (a.City || '').trim(),
+ county: (a.County || COUNTY_NAME[COUNTY] || '').trim(),
+ state: 'AZ',
+ zip: (a.Zip || '').trim(),
+ phone,
+ website: '', // directory does not expose a website field
+ market: 'Arizona',
+ commercial_flag: 'true', // real-estate practice = CRE-relevant
+ within_300mi: 'true', // Maricopa/Scottsdale within 300mi-of-Scottsdale scope
+ source_url: DIR_URL,
+ };
+}
+
+async function main() {
+ const specLabel = SPEC === 'RE' ? 'Real Estate Law' : SPEC;
+ console.log(`[az_state_bar] spec=${SPEC} (${specLabel}) county=${COUNTY} (${COUNTY_NAME[COUNTY] || '?'}) pageSize=${PAGE_SIZE}`);
+
+ const browser = await chromium.launch({ headless: true });
+ const ctx = await browser.newContext();
+ const page = await ctx.newPage();
+ page.setDefaultTimeout(60000);
+
+ // Establish origin/referer by loading the real directory page (also proves basic search isn't login-walled).
+ await page.goto(DIR_URL, { waitUntil: 'domcontentloaded' });
+ await sleep(1500);
+
+ // Extract the PUBLIC search-widget auth (userid/password) live from the page — a
+ // public token the page ships for its own directory search; runtime-fetching it
+ // keeps the secret out of git and survives rotation.
+ const html = await page.content();
+ // The widget sets them via: setRequestHeader("userid","publictools"); setRequestHeader("password","<token>")
+ const uid = (html.match(/setRequestHeader\(\s*["']userid["']\s*,\s*["']([^"']+)["']/i) || [])[1];
+ const pw = (html.match(/setRequestHeader\(\s*["']password["']\s*,\s*["']([^"']+)["']/i) || [])[1];
+ if (uid) HEADERS.userid = uid;
+ if (pw) HEADERS.password = pw;
+ if (!HEADERS.userid || !HEADERS.password) {
+ console.error('[az_state_bar] FATAL: could not extract public search auth from the directory page — the widget markup likely changed');
+ process.exit(1);
+ }
+
+ const bodyBase = {
+ Type: 'Member', Firm: '', FuzzySearch: false, IncludeDeceased: false,
+ FirstName: '', LastName: '', City: '', State: '', Zip: '',
+ County: COUNTY, LanguageCode: '', Section: '', LegalNeed: '',
+ Specialization: SPEC, JurisdictionCode: '', LawSchool: '',
+ };
+
+ const seen = new Set(); // dedupe by EntityNumber
+ const rows = [];
+ let total = null;
+ let pageNo = 1;
+
+ while (true) {
+ const url = `${API}?PageSize=${PAGE_SIZE}&Page=${pageNo}&Shuffle=false&Seed=null`;
+ const res = await page.evaluate(async ({ url, headers, body }) => {
+ const r = await fetch(url, { method: 'POST', headers, body: JSON.stringify(body) });
+ let text = '';
+ try { text = await r.text(); } catch { /* */ }
+ return { status: r.status, text };
+ }, { url, headers: HEADERS, body: bodyBase });
+
+ if (res.status !== 200) {
+ console.error(`[az_state_bar] page ${pageNo} HTTP ${res.status} — stopping. Body: ${res.text?.slice(0, 300)}`);
+ break;
+ }
+ let json;
+ try { json = JSON.parse(res.text); } catch (e) {
+ console.error(`[az_state_bar] page ${pageNo} JSON parse fail — stopping.`); break;
+ }
+ if (!json.IsSuccess || !json.Result) {
+ console.error(`[az_state_bar] page ${pageNo} not success (${json.Message || json.Error}) — stopping.`); break;
+ }
+ if (total === null) { total = json.Result.TotalCount; console.log(`[az_state_bar] TotalCount = ${total}`); }
+
+ const batch = json.Result.Results || [];
+ if (!batch.length) break;
+
+ for (const m of batch) {
+ const id = m.EntityNumber ?? `${m.BarNumber}-${m.LastName}`;
+ if (seen.has(id)) continue;
+ seen.add(id);
+ rows.push(mapRow(m));
+ }
+ console.log(`[az_state_bar] page ${pageNo}: +${batch.length} (running ${rows.length}/${total})`);
+
+ if (rows.length >= (total ?? 0) || batch.length < PAGE_SIZE) break;
+ pageNo += 1;
+ await sleep(POLITE_MS); // polite ~1 req/s
+ }
+
+ await browser.close();
+
+ // write CSV
+ mkdirSync(dirname(OUT), { recursive: true });
+ const lines = [COLS.join(',')];
+ for (const r of rows) lines.push(COLS.map((c) => csvCell(r[c])).join(','));
+ writeFileSync(OUT, lines.join('\n') + '\n');
+
+ console.log(`[az_state_bar] wrote ${rows.length} rows -> ${OUT}`);
+ if (total !== null && rows.length !== total) {
+ console.warn(`[az_state_bar] NOTE: rows (${rows.length}) != TotalCount (${total}).`);
+ }
+}
+
+main().catch((e) => { console.error('[az_state_bar] FATAL', e); process.exit(1); });
← cd9b7e6 auto-data-snapshot: 2026-08-06T14:29:57 (1 data files) — dat
·
back to Rentv Licensed Targets
·
A2: CA DOI title-insurer scraper (CDI APEX) — resilient (ins 1b043a9 →