`scripts/discover-broker-endpoint.js`

63 lines
// discover-broker-endpoint.js — DEFINITIVE one-session discovery of Crexi's per-broker listings API.
// Navigates real broker PROFILE pages by publicProfileId slug (crexi.com/broker/<slug>) and logs
// EVERY api.crexi.com response: method, path, top keys, and whether it carries sale[]/lease[]/assets.
// Read-only; writes data/raw/discover-broker-endpoint.json. ~$0.04 (1 Browserbase session).
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const ROOT = path.join(__dirname, '..');
// Real Crexi profile paths: crexi.com/profile/<firstname-lastname>-<publicProfileId>.
// Add via env: PROFILES="george-ouzounian-georgeouzo,ash-ghavami-ashgha"
const PROFILES = (process.env.PROFILES || 'george-ouzounian-georgeouzo,ash-ghavami-ashgha,steven-roberts-stevenrober').split(',').map(s => s.trim()).filter(Boolean);

(async () => {
  const hits = [];
  const postBodies = [];
  let browser, auth = null;
  try {
    const bb = new Browserbase({ apiKey: get(env, 'BROWSERBASE_API_KEY') });
    const s = await bb.sessions.create({ projectId: get(env, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    console.log('bb session', s.id);
    browser = await chromium.connectOverCDP(s.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);
    page.on('request', r => {
      const h = r.headers(); if (!auth && h.authorization && /bearer/i.test(h.authorization) && r.url().includes('api.crexi.com')) auth = h.authorization;
      if (r.method() === 'POST' && /api\.crexi\.com\/assets\/(geo\/)?search/.test(r.url())) { const pd = r.postData(); if (pd) postBodies.push({ url: r.url(), body: pd }); }
    });
    page.on('response', async (resp) => {
      const u = resp.url();
      if (!u.includes('api.crexi.com')) return;
      if ((resp.headers()['content-type'] || '').indexOf('json') < 0) return;
      let j; try { j = await resp.json(); } catch { return; }
      let p; try { p = new URL(u).pathname + (new URL(u).search ? '?' + new URL(u).search.slice(1, 60) : ''); } catch { p = u; }
      const keys = Array.isArray(j) ? '[array]' : Object.keys(j || {}).slice(0, 16);
      const has = (k) => Array.isArray(j?.[k]) && j[k].length;
      const listLen = (Array.isArray(j) ? j.length : ([].concat(j.data || [], j.assets || [], j.results || [], j.items || [], j.sale || [], j.lease || [], j.forSale || [])).length);
      hits.push({ method: resp.request().method(), path: p, keys, sale: has('sale'), lease: has('lease'), assets: has('assets') || has('data') || has('results'), listLen });
    });
    for (const slug of PROFILES) {
      const url = `https://www.crexi.com/profile/${slug}`;
      console.log('→', url);
      try { await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.waitForTimeout(6000); await page.mouse.wheel(0, 4000).catch(() => {}); await page.waitForTimeout(4000); } catch (e) { console.log('  nav err', e.message.split('\n')[0]); }
    }
  } catch (e) { console.error('FATAL', e.message); }
  finally { if (browser) try { await browser.close(); } catch {} }

  hits.sort((a, b) => b.listLen - a.listLen);
  fs.writeFileSync(path.join(ROOT, 'data', 'raw', 'discover-broker-endpoint.json'), JSON.stringify({ slugs: PROFILES, bearer: !!auth, postBodies, hits }, null, 2));
  console.log('\n/assets/search POST bodies captured:', postBodies.length);
  postBodies.slice(0, 4).forEach(b => console.log('  ', b.url.split('crexi.com')[1], '→', b.body.slice(0, 400)));
  console.log('\nendpoints carrying listings (sale/lease/assets), richest first:');
  for (const h of hits.filter(x => x.sale || x.lease || x.assets || x.listLen > 1).slice(0, 15))
    console.log(`  [${h.method}] ${h.path}  len=${h.listLen} sale=${h.sale} lease=${h.lease}  keys=${Array.isArray(h.keys) ? h.keys.join(',') : h.keys}`);
  console.log(`\ntotal api.crexi.com JSON responses: ${hits.length}  |  bearer: ${auth ? 'yes' : 'no'}`);
  console.log('💰 ~$0.04 (1 session). Full dump: data/raw/discover-broker-endpoint.json');
  process.exit(0);
})();