`scripts/scrape-crexi.js`

115 lines
// scrape-crexi.js — Drive a real Browserbase cloud browser to Crexi, capture the
// PROPERLY-FILTERED /assets/search XHR JSON (the public api.crexi.com feed ignores
// our body when called raw; from a real session it returns city/type/price-scoped data).
//
// Strategy: navigate Crexi search for each SFV city, listen for api.crexi.com responses
// carrying a `data[]` array, dedupe by asset id, save everything to data/raw/.
// Also dump the request postData of /assets/search so we learn the real filter schema.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;

const skillEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (txt, k) => (txt.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const PROJECT = get(skillEnv, 'BROWSERBASE_PROJECT_ID');
const KEY = get(skillEnv, 'BROWSERBASE_API_KEY');

const RAW = path.join(__dirname, '..', 'data', 'raw');
fs.mkdirSync(RAW, { recursive: true });

// SFV cities (full San Fernando Valley coverage)
const SFV_CITIES = [
  'Van Nuys', 'Sherman Oaks', 'North Hollywood', 'Encino', 'Reseda', 'Northridge',
  'Canoga Park', 'Woodland Hills', 'Tarzana', 'Panorama City', 'Sun Valley', 'Pacoima',
  'Granada Hills', 'Chatsworth', 'Winnetka', 'Sylmar', 'Mission Hills', 'Valley Village',
  'Studio City', 'Arleta', 'Lake Balboa', 'West Hills', 'Porter Ranch', 'Sunland', 'Tujunga'
];

const TYPE_SLUGS = { Multifamily: 'multifamily', Retail: 'retail' };

// CRE_LIMIT=N caps cities (probe mode); CRE_TYPES=Multifamily limits types
const LIMIT = parseInt(process.env.CRE_LIMIT || '0', 10);
const CITIES = LIMIT ? SFV_CITIES.slice(0, LIMIT) : SFV_CITIES;
const TYPES = process.env.CRE_TYPES
  ? Object.fromEntries(Object.entries(TYPE_SLUGS).filter(([k]) => process.env.CRE_TYPES.includes(k)))
  : TYPE_SLUGS;

(async () => {
  const bb = new Browserbase({ apiKey: KEY });
  const session = await bb.sessions.create({
    projectId: PROJECT,
    browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 900 } }
  });
  console.log('bb session', session.id, '| live:', session.debuggerFullscreenUrl || '(n/a)');
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0];
  const page = ctx.pages()[0] || await ctx.newPage();
  page.setDefaultTimeout(60000);

  const assets = new Map();           // id -> asset object
  const searchBodies = [];            // captured request postData schemas
  let respCount = 0;

  page.on('request', (req) => {
    const u = req.url();
    if (u.includes('api.crexi.com') && u.includes('search') && req.method() === 'POST') {
      const pd = req.postData();
      if (pd) searchBodies.push({ url: u, body: pd });
    }
  });

  page.on('response', async (resp) => {
    const u = resp.url();
    if (!u.includes('api.crexi.com')) return;
    if (resp.request().resourceType() === 'preflight') return;
    try {
      const ct = resp.headers()['content-type'] || '';
      if (!ct.includes('json')) return;
      const j = await resp.json();
      const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || null);
      if (Array.isArray(arr) && arr.length && arr[0] && (arr[0].id || arr[0].assetId)) {
        respCount++;
        let added = 0;
        for (const a of arr) {
          const id = a.id || a.assetId;
          if (id && !assets.has(id)) { assets.set(id, a); added++; }
        }
        console.log(`  xhr#${respCount} ${u.split('?')[0].split('crexi.com')[1]} +${added} (total ${assets.size})`);
      }
    } catch (_) { /* non-json or consumed */ }
  });

  async function visit(url, label) {
    try {
      console.log(`\n→ ${label}: ${url}`);
      await page.goto(url, { waitUntil: 'domcontentloaded' });
      await page.waitForTimeout(5000);
      // scroll to trigger lazy XHR / pagination
      for (let i = 0; i < 3; i++) {
        await page.mouse.wheel(0, 4000).catch(() => {});
        await page.waitForTimeout(2500);
      }
    } catch (e) { console.log(`  ! ${label} error: ${e.message.split('\n')[0]}`); }
  }

  // 1) Warm a session on a search page so cookies/headers are set
  await visit('https://www.crexi.com/properties?types[]=Multifamily', 'warm: MF search');

  // 2) Per-city, per-type city landing pages (these fire city-scoped /assets/search)
  for (const [type, slug] of Object.entries(TYPES)) {
    for (const city of CITIES) {
      const citySlug = city.replace(/\s+/g, '-');
      const url = `https://www.crexi.com/properties/CA/${encodeURIComponent(citySlug)}/${slug}`;
      await visit(url, `${type} / ${city}`);
      fs.writeFileSync(path.join(RAW, 'assets.json'), JSON.stringify([...assets.values()], null, 2));
    }
  }

  fs.writeFileSync(path.join(RAW, 'assets.json'), JSON.stringify([...assets.values()], null, 2));
  fs.writeFileSync(path.join(RAW, 'search-bodies.json'), JSON.stringify(searchBodies.slice(0, 20), null, 2));
  console.log(`\n=== DONE: ${assets.size} unique assets captured, ${searchBodies.length} search bodies ===`);
  await browser.close().catch(() => {});
  process.exit(0);
})().catch(e => { console.error('FATAL', e); process.exit(1); });