`src/scripts/probe_calbar.ts`

73 lines
/**
 * Network probe — open one CalBar Detail/{id} page in Playwright and log every
 * XHR/fetch request and response so we can identify the data endpoint that
 * powers the SPA. Once found we'll replay it via plain fetch for all ~330k
 * bar numbers without keeping a browser session per request.
 */
import { chromium } from 'playwright';

const SAMPLE_BAR_NUMBER = process.argv[2] || '29960';
const URL = `https://apps.calbar.ca.gov/attorney/Licensee/Detail/${SAMPLE_BAR_NUMBER}`;

(async () => {
  const browser = await chromium.launch({ headless: true });
  try {
  const ctx = await browser.newContext({
    userAgent: 'LawyerDirectoryBuilder/0.1 (research; contact: steveabramsdesigns@gmail.com)',
  });
  const page = await ctx.newPage();

  const requests: Array<{ url: string; method: string; resourceType: string; headers: Record<string,string> }> = [];
  page.on('request', req => {
    const u = req.url();
    if (u.includes('calbar') || u.includes('apps.calbar')) {
      requests.push({ url: u, method: req.method(), resourceType: req.resourceType(), headers: req.headers() });
    }
  });

  const responses: Array<{ url: string; status: number; ct: string; bodyPreview: string; bodyLen: number }> = [];
  page.on('response', async res => {
    const u = res.url();
    if (!u.includes('calbar') && !u.includes('apps.calbar')) return;
    const ct = res.headers()['content-type'] || '';
    if (!ct.includes('json') && !ct.includes('xml') && !ct.includes('javascript')) return;
    try {
      const body = await res.text();
      responses.push({
        url: u, status: res.status(), ct,
        bodyPreview: body.slice(0, 600),
        bodyLen: body.length,
      });
    } catch {}
  });

  console.log(`probing ${URL}…`);
  await page.goto(URL, { waitUntil: 'networkidle', timeout: 30000 });
  await page.waitForTimeout(2000); // catch lazy XHRs

  console.log('\n=== JSON/XML/JS responses (calbar domain) ===');
  for (const r of responses) {
    console.log(`\n${r.status}  ${r.url}`);
    console.log(`  content-type: ${r.ct}  size: ${r.bodyLen}`);
    console.log(`  body preview:\n  ${r.bodyPreview.replace(/\n/g, '\n  ').slice(0, 500)}`);
  }

  console.log('\n=== content of the rendered page ===');
  const title = await page.title();
  console.log(`title: ${title}`);
  // Grab the innerText of the rendered detail panel if present
  const text = await page.evaluate(`(function(){
    var sel = ['#moduleAttorneySearchList', '#detail', '.lic-info', '.attorney-detail', 'main'];
    for (var i=0;i<sel.length;i++) {
      var el = document.querySelector(sel[i]);
      if (el && el.innerText) return { selector: sel[i], text: el.innerText.slice(0, 800) };
    }
    return { selector: 'body', text: (document.body.innerText||'').slice(0, 800) };
  })()`) as any;
  console.log(`from "${text.selector}":\n${text.text}`);

  } finally {
    await browser.close();
  }
})().catch(e => { console.error('probe fail:', e); process.exit(1); });