`scripts/discover-firm.js`

95 lines
// discover-firm.js — generalized schema cracker for any firm in sources/firms.js. Warms the firm's
// search page in one Browserbase session and logs EVERY JSON response on its apiHosts: method, path,
// item-count, top-level keys, and a sample asset. Use this to confirm a 'needs-discovery' firm's API
// shape, then either trust genericExtract or write a precise extract() in firms.js and flip it to
// 'verified'. This is the same technique that originally cracked Crexi (discover.js / discover2.js).
//
//   node scripts/discover-firm.js marcusmillichap            # default first SFV city
//   CC_NAV="Van Nuys" node scripts/discover-firm.js colliers
//
// COST: ~$0.03–0.05 (one Browserbase session). Printed at the end. READ-ONLY — writes nothing to
// listings.json; output goes to data/raw/discover-<firm>.json for inspection.

'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const { TYPES, byKey, genericExtract } = require('./sources/firms');

const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();

const KEY = process.argv[2];
const firm = byKey(KEY);
if (!firm) { console.error('usage: node scripts/discover-firm.js <firmKey>   e.g. marcusmillichap | lee-associates | colliers | cbre | jll'); process.exit(2); }
const CITY = (process.env.CC_NAV || 'Van Nuys').split(',')[0].trim();

(async () => {
  const hits = [];
  let browser;
  try {
    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
    const session = await bb.sessions.create({
      projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'),
      browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } }
    });
    console.log('bb session', session.id, '| firm', firm.key, '| city', CITY);
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(60000);

    page.on('response', async (resp) => {
      const url = resp.url();
      if (!firm.apiHosts.some(h => url.includes(h))) return;
      const ct = resp.headers()['content-type'] || '';
      if (!/json/i.test(ct)) return;
      let j; try { j = await resp.json(); } catch { return; }
      let sample = null, count = 0, generic = 0;
      try {
        const g = genericExtract(j); generic = g.length; sample = g[0] || null;
        const arr = Array.isArray(j) ? j : (j.data || j.results || j.assets || j.items || []);
        count = Array.isArray(arr) ? arr.length : 0;
      } catch {}
      let u; try { u = new URL(url); } catch { u = { pathname: url, hostname: '' }; }
      hits.push({
        method: resp.request().method(), host: u.hostname, path: u.pathname,
        topKeys: Array.isArray(j) ? '[array]' : Object.keys(j || {}).slice(0, 14),
        rawCount: count, genericExtracted: generic, sample
      });
    });

    for (const [slug] of TYPES) {
      for (const url of firm.navUrls(CITY, slug)) {
        console.log('→', url);
        try {
          await page.goto(url, { waitUntil: 'domcontentloaded' });
          await page.waitForTimeout(8000);
          await page.mouse.wheel(0, 3000).catch(() => {});
          await page.waitForTimeout(4000);
        } catch (e) { console.log('nav err', String(e.message).split('\n')[0]); }
      }
    }
  } catch (e) {
    console.error('FATAL', e.message);
  } finally {
    if (browser) await browser.close().catch(() => {});
  }

  hits.sort((a, b) => b.genericExtracted - a.genericExtracted);
  const outFile = path.join(ROOT, 'data', 'raw', `discover-${firm.key}.json`);
  fs.writeFileSync(outFile, JSON.stringify({ firm: firm.key, city: CITY, capturedResponses: hits.length, hits }, null, 2));

  console.log(`\n${firm.name}: ${hits.length} JSON responses on ${firm.apiHosts.join('/')}`);
  for (const h of hits.slice(0, 12)) {
    console.log(`  [${h.method}] ${h.path}  raw=${h.rawCount} generic=${h.genericExtracted}  keys=${Array.isArray(h.topKeys) ? h.topKeys.join(',') : h.topKeys}`);
  }
  const best = hits.find(h => h.genericExtracted > 0);
  console.log(best
    ? `\n✓ genericExtract already pulls ${best.genericExtracted} assets from ${best.path} — sample: ${JSON.stringify(best.sample)}\n  → safe to run: CC_FIRMS=${firm.key} node scripts/refresh-all.js`
    : `\n✗ genericExtract found 0 assets — inspect ${outFile} and write a precise extract() in sources/firms.js for "${firm.key}".`);
  console.log('Cost: ~$0.03–0.05 (1 Browserbase session).');
})();