`scripts/discover.js`

88 lines
// discover.js — one warm Crexi session; (1) log every POST the UI makes to api.crexi.com
// with path+item-count, (2) run in-page fetch experiments against /universal-search/search
// to find the filter schema that returns SFV multifamily listings.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const skillEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const bb = new Browserbase({ apiKey: get(skillEnv, 'BROWSERBASE_API_KEY') });
const RAW = path.join(__dirname, '..', 'data', 'raw');

(async () => {
  const session = await bb.sessions.create({ projectId: get(skillEnv, 'BROWSERBASE_PROJECT_ID'),
    browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 900 } } });
  console.log('bb session', session.id);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0];
  const page = ctx.pages()[0] || await ctx.newPage();
  page.setDefaultTimeout(60000);

  const posts = [];
  page.on('response', async (resp) => {
    const u = new URL(resp.url());
    if (!u.hostname.includes('api.crexi.com')) return;
    if (resp.request().method() !== 'POST') return;
    try {
      const j = await resp.json();
      const arr = Array.isArray(j) ? j : (j.data || j.results || j.assets || j.items || []);
      posts.push({ path: u.pathname, body: resp.request().postData(), count: Array.isArray(arr) ? arr.length : 0,
        topKeys: Array.isArray(j) ? '[array]' : Object.keys(j).slice(0, 12),
        sampleLoc: (arr[0] && (arr[0].address || (arr[0].locations && arr[0].locations[0]))) || null });
    } catch (_) {}
  });

  console.log('→ warming crexi.com search');
  await page.goto('https://www.crexi.com/properties', { waitUntil: 'domcontentloaded' }).catch(()=>{});
  await page.waitForTimeout(8000);
  await page.mouse.wheel(0, 3000).catch(()=>{}); await page.waitForTimeout(4000);

  // Save what the UI itself POSTed (ground-truth schema)
  fs.writeFileSync(path.join(RAW, 'ui-posts.json'), JSON.stringify(posts, null, 2));
  console.log(`UI made ${posts.length} POSTs to api.crexi.com:`);
  posts.forEach(p => console.log(`  ${p.path}  count=${p.count}  keys=${JSON.stringify(p.topKeys)}`));

  // In-page experiments against /universal-search/search
  const SFV_BBOX = { north: 34.33, south: 34.12, east: -118.28, west: -118.68 };
  const experiments = [
    { tag: 'empty', filters: {} },
    { tag: 'types-Multifamily', filters: { types: ['Multifamily'] } },
    { tag: 'assetTypes', filters: { assetTypes: ['Multifamily'] } },
    { tag: 'propertyTypes', filters: { propertyTypes: ['Multifamily'] } },
    { tag: 'bbox-mapBounds', filters: { mapBounds: SFV_BBOX } },
    { tag: 'bbox-geo', filters: { boundingBox: { neLat: SFV_BBOX.north, neLon: SFV_BBOX.east, swLat: SFV_BBOX.south, swLon: SFV_BBOX.west } } },
    { tag: 'city', filters: { 'address.city': ['Van Nuys'] } },
    { tag: 'locations', filters: { locations: [{ city: 'Van Nuys', state: 'CA' }] } },
  ];
  const results = [];
  for (const ex of experiments) {
    const r = await page.evaluate(async ({ filters }) => {
      const attempts = [];
      for (const shape of [
        { filters, searchTypes: ['Sales'], size: 5 },
        { filters, searchTypes: ['Sales'], size: 5, from: 0 },
        { filters, searchTypes: ['Sales'], pageSize: 5, page: 1 },
      ]) {
        try {
          const res = await fetch('https://api.crexi.com/universal-search/search', {
            method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(shape) });
          const txt = await res.text();
          let j; try { j = JSON.parse(txt); } catch { j = null; }
          const arr = j ? (Array.isArray(j) ? j : (j.data || j.results || j.assets || j.items || [])) : [];
          attempts.push({ status: res.status, count: Array.isArray(arr) ? arr.length : 0,
            firstCity: arr[0] && (arr[0].locations?.[0]?.city || arr[0].address?.city || arr[0].city) || null,
            err: (j && j.message) || (txt.slice(0, 120)) });
        } catch (e) { attempts.push({ error: String(e).slice(0, 100) }); }
      }
      return attempts;
    }, ex);
    const best = r.find(a => a.count > 0) || r[0];
    console.log(`  [${ex.tag}] ${JSON.stringify(best)}`);
    results.push({ tag: ex.tag, filters: ex.filters, attempts: r });
  }
  fs.writeFileSync(path.join(RAW, 'experiments.json'), JSON.stringify(results, null, 2));
  await browser.close().catch(()=>{});
  process.exit(0);
})().catch(e => { console.error('FATAL', e); process.exit(1); });