`scripts/probe-brokers.js`

68 lines
// probe-brokers.js — navigate real Crexi DETAIL pages and deep-scan every api.crexi.com JSON response
// for broker/agent data. Writes data/raw/broker-probe.json with the endpoint(s) that carry brokers and
// a sample. One Browserbase session (~$0.04). NODE_PATH must point at the browserbase skill.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const ROOT = path.join(__dirname, '..');

const DETAIL_URLS = [
  'https://www.crexi.com/properties/2560155/california-6852-claire-ave-reseda-ca-91335-3840',
  'https://www.crexi.com/properties/2448930/california-17852-saticoy-street'
];

// Deep-scan an object for broker/agent-ish keys at any depth; return matched {path:value} pairs.
function scanBroker(o, prefix = '', hits = {}, depth = 0) {
  if (!o || typeof o !== 'object' || depth > 6) return hits;
  for (const k of Object.keys(o)) {
    const p = prefix ? prefix + '.' + k : k;
    if (/^(brokers?|agents?|contacts?|brokerage|salesPerson|advisor|team|listedBy|firm|company)$/i.test(k)) {
      hits[p] = JSON.stringify(o[k]).slice(0, 600);
    }
    if (o[k] && typeof o[k] === 'object') scanBroker(o[k], p, hits, depth + 1);
  }
  return hits;
}

(async () => {
  const endpoints = {};   // pathname -> { keys, brokerHits, sampleSaved }
  let savedSample = null;
  let browser;
  try {
    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
    const session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    console.log('bb session', session.id);
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);
    page.on('response', async (resp) => {
      const u = resp.url(); if (!u.includes('api.crexi.com')) return;
      let j; try { j = await resp.json(); } catch { return; }
      const pathname = new URL(u).pathname.replace(/\/\d+/g, '/<id>');
      const hits = scanBroker(j);
      const e = endpoints[pathname] || (endpoints[pathname] = { keys: Array.isArray(j) ? '[array]' : Object.keys(j).slice(0, 20), brokerPaths: {} });
      Object.assign(e.brokerPaths, hits);
      if (Object.keys(hits).length && !savedSample) savedSample = { url: pathname, body: j };
    });
    for (const url of DETAIL_URLS) {
      console.log('→', url);
      await page.goto(url, { waitUntil: 'domcontentloaded' }).catch(e => console.log('  nav', e.message.split('\n')[0]));
      await page.waitForTimeout(7000);
      await page.mouse.wheel(0, 4000).catch(() => {}); await page.waitForTimeout(3000);
    }
  } catch (e) { console.error('FATAL', e.message); } finally { if (browser) await browser.close().catch(() => {}); }

  fs.writeFileSync(path.join(ROOT, 'data', 'raw', 'broker-probe.json'), JSON.stringify({ endpoints, savedSample }, null, 2));
  console.log('\n=== endpoints seen ===');
  for (const [p, e] of Object.entries(endpoints)) {
    const bk = Object.keys(e.brokerPaths);
    console.log(`${p}  ${bk.length ? '★ BROKER: ' + bk.join(', ') : '(no broker fields)'}`);
  }
  console.log('\nfull dump → data/raw/broker-probe.json');
})();