`crawl/crexi-enrich-recon.js`

43 lines
'use strict';
// Recon (~$0.30): find Crexi's broker-profile endpoint + whether contact (phone/email/site) is
// exposed. Loads a broker profile, captures api.crexi.com XHRs, and probes candidate endpoints.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default || require('@browserbasehq/sdk');
const envTxt = fs.readFileSync(path.join(process.env.HOME, '.claude/skills/browserbase/.env'), 'utf8');
const API_KEY = (envTxt.match(/BROWSERBASE_API_KEY=(.*)/) || [])[1]?.trim();
const PROJECT = (envTxt.match(/BROWSERBASE_PROJECT_ID=(.*)/) || [])[1]?.trim();
const ID = process.argv[2] || '806';

(async () => {
  const t0 = Date.now();
  const bb = new Browserbase({ apiKey: API_KEY });
  const session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0]; const page = ctx.pages()[0] || await ctx.newPage();
  page.setDefaultTimeout(45000);
  const seen = new Set();
  page.on('response', async (res) => { const u = res.url(); if (u.includes('api.crexi.com') && /broker|user|contact|profile/i.test(u)) seen.add(u.split('?')[0]); });
  // try to reach a broker profile via search page first (to get a valid URL pattern)
  for (const url of [`https://www.crexi.com/brokers/${ID}`, `https://www.crexi.com/broker/${ID}`]) {
    try { await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 35000 }); await page.waitForTimeout(5000); await page.mouse.wheel(0, 2000).catch(()=>{}); await page.waitForTimeout(2500); } catch {}
  }
  // probe candidate API endpoints in-page (authorized via cookies)
  const probe = await page.evaluate(async (id) => {
    const cands = [`/brokers/${id}`, `/users/${id}`, `/brokers/${id}/profile`, `/brokers/${id}/contact-info`, `/broker-profiles/${id}`, `/users/${id}/profile`];
    const out = [];
    for (const c of cands) {
      try { const r = await fetch('https://api.crexi.com' + c, { headers: {} });
        const body = r.ok ? await r.text() : ''; out.push({ c, status: r.status, hasEmail: /email/i.test(body), hasPhone: /phone/i.test(body), snip: body.slice(0, 260) }); }
      catch (e) { out.push({ c, err: String(e).slice(0, 40) }); }
    }
    return out;
  }, ID);
  await browser.close().catch(()=>{});
  console.log('cost: $' + ((Date.now()-t0)/60000*0.15).toFixed(2));
  console.log('profile XHRs seen:', [...seen].join(', ') || 'none');
  console.log('=== endpoint probes ===');
  probe.forEach(p => console.log(`[${p.status||p.err}] ${p.c}  email:${p.hasEmail} phone:${p.hasPhone}\n   ${(p.snip||'').replace(/\s+/g,' ').slice(0,200)}`));
})();