`crawl/crexi-recon.js`

40 lines
'use strict';
// Recon (~$0.30): capture Crexi's bearer token + discover the property-search and/or
// broker-directory API endpoints so the national broker fetch runs with a known plan.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default || require('@browserbasehq/sdk');
const envTxt = fs.readFileSync(path.join(process.env.HOME, '.claude/skills/browserbase/.env'), 'utf8');
const API_KEY = (envTxt.match(/BROWSERBASE_API_KEY=(.*)/) || [])[1]?.trim();
const PROJECT = (envTxt.match(/BROWSERBASE_PROJECT_ID=(.*)/) || [])[1]?.trim();

(async () => {
  const t0 = Date.now();
  const bb = new Browserbase({ apiKey: API_KEY });
  const session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0]; const page = ctx.pages()[0] || await ctx.newPage();
  page.setDefaultTimeout(45000);
  let auth = null; const apis = new Map();
  page.on('request', req => { const h = req.headers(); if (!auth && h.authorization && /bearer/i.test(h.authorization) && req.url().includes('api.crexi.com')) auth = h.authorization; });
  page.on('response', async (res) => {
    const u = res.url(); if (!u.includes('api.crexi.com')) return;
    const key = u.split('?')[0];
    if (!apis.has(key)) { let sample = ''; try { const t = await res.text(); sample = t.slice(0, 240); } catch {} apis.set(key, { status: res.status, sample, full: u }); }
  });
  const visited = [];
  for (const url of ['https://www.crexi.com/properties/TX/Houston', 'https://www.crexi.com/brokers']) {
    try { await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 40000 }); await page.waitForTimeout(6000); await page.mouse.wheel(0, 3000).catch(()=>{}); await page.waitForTimeout(3000); visited.push(url + ' OK'); }
    catch (e) { visited.push(url + ' ERR ' + String(e).slice(0,50)); }
  }
  await browser.close().catch(()=>{});
  const mins = (Date.now()-t0)/60000;
  console.log('visited:', visited.join(' | '));
  console.log('bearer captured:', auth ? 'YES' : 'no');
  console.log('cost: $' + (mins*0.15).toFixed(2), `(${mins.toFixed(1)} min)`);
  console.log('=== api.crexi.com endpoints seen ===');
  for (const [k, v] of apis) console.log(`[${v.status}] ${k}\n   full: ${v.full.slice(0,160)}\n   sample: ${v.sample.replace(/\s+/g,' ').slice(0,180)}`);
  fs.writeFileSync(path.join(__dirname, '..', 'data', 'crexi-recon.json'), JSON.stringify({ auth: auth?auth.slice(0,20)+'…':null, apis: [...apis] }, null, 1));
})();