`src/scripts/inspect_justia.ts`

55 lines
/**
 * One-shot probe — fetches a single Justia city page, checks robots, and
 * dumps a structural summary so we can pick the right CSS selectors before
 * writing the real crawler.
 */
import 'dotenv/config';
import { load } from 'cheerio';
import { fetchCompliant, isAllowed } from '../lib/compliance.ts';

const URL = process.argv[2] || 'https://www.justia.com/lawyers/california/beverly-hills';

(async () => {
  console.log(`probing ${URL}`);
  console.log(`robots allowed: ${await isAllowed(URL)}`);

  const res = await fetchCompliant(URL);
  console.log(`status: ${res.status}  content-type: ${res.headers.get('content-type')}`);
  const html = await res.text();
  console.log(`bytes: ${html.length}`);

  const $ = load(html);
  console.log(`<title>: ${$('title').text().trim()}`);

  // Common Justia patterns
  const candidates = [
    '.lawyer-card', '.lawyer', '.lawyers li', '.search-results-list li',
    'div[itemtype*="LegalService"]', 'div[itemtype*="Person"]',
    '.has-img.iblock', '.iblock',
    '.facet-doc', '.entry', '.law-firm', '.firm',
  ];
  for (const sel of candidates) {
    const n = $(sel).length;
    if (n > 0) console.log(`  ${sel}  → ${n} matches`);
  }

  // Show first 3 anchors that look like lawyer/firm profiles
  console.log('\nfirst 6 profile-ish links:');
  const seen = new Set<string>();
  $('a[href]').each((_, el) => {
    const h = $(el).attr('href') || '';
    if (!h.includes('/lawyers/') && !h.includes('/lawfirm/')) return;
    if (h === '/lawyers/' || /\/(?:bookmark|share|review|save|map)/i.test(h)) return;
    if (seen.has(h)) return;
    seen.add(h);
    if (seen.size <= 6) console.log(`  ${$(el).text().trim().slice(0,60)}  →  ${h}`);
  });

  // Show first card-ish block
  const card = $('.lawyer-card, .iblock, .has-img').first();
  if (card.length) {
    console.log('\nfirst card outerHTML (truncated 1.2k):');
    console.log($.html(card).slice(0, 1200));
  }
})();