`scripts/capture-firm-body.js`

65 lines
// capture-firm-body.js — dump the FULL raw JSON bodies of a firm's listing API so we can write a
// precise extract() (docs/SOURCING.md Phase 2). Unlike discover-firm.js (metadata only), this saves
// the actual response payloads to data/raw/body-<firm>-<i>.json for the biggest arrays seen.
//
//   NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/capture-firm-body.js colliers
// COST: ~$0.04 per firm (1 Browserbase session). Printed at end.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const { byKey, TYPES } = require('./sources/firms');
const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();

const KEY = process.argv[2];
const firm = byKey(KEY);
if (!firm) { console.error('usage: node scripts/capture-firm-body.js <firmKey>'); process.exit(2); }
const CITY = (process.env.CC_NAV || 'Van Nuys').split(',')[0].trim();

(async () => {
  const bodies = [];   // {path, count, body}
  let browser;
  try {
    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
    const session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    console.log('bb session', session.id, '| firm', firm.key, '| city', CITY);
    browser = await chromium.connectOverCDP(session.connectUrl);
    const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();
    page.setDefaultTimeout(60000);

    page.on('response', async (resp) => {
      const url = resp.url();
      if (!firm.apiHosts.some(h => url.includes(h))) return;
      if (!/json/i.test(resp.headers()['content-type'] || '')) return;
      let j; try { j = await resp.json(); } catch { return; }
      // Keep bodies that plausibly carry a listings array anywhere in them.
      const s = JSON.stringify(j);
      if (s.length < 400) return;                 // skip tiny lookups
      let u; try { u = new URL(url); } catch { u = { pathname: url }; }
      bodies.push({ path: u.pathname, bytes: s.length, body: j });
    });

    for (const [slug] of TYPES.slice(0, 2)) {     // 2 asset classes is enough to sample the shape
      for (const url of firm.navUrls(CITY, slug)) {
        console.log('→', url);
        try { await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.waitForTimeout(9000); await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(4000); }
        catch (e) { console.log('nav err', String(e.message).split('\n')[0]); }
      }
    }
  } catch (e) { console.error('FATAL', e.message); }
  finally { if (browser) await browser.close().catch(() => {}); }

  // Save the biggest bodies (most likely the listings payloads).
  bodies.sort((a, b) => b.bytes - a.bytes);
  const saved = bodies.slice(0, 4);
  saved.forEach((b, i) => {
    const fp = path.join(ROOT, 'data', 'raw', `body-${firm.key}-${i}.json`);
    fs.writeFileSync(fp, JSON.stringify(b.body, null, 2));
    console.log(`saved ${fp}  (${b.path}, ${(b.bytes / 1024).toFixed(1)}kb)`);
  });
  console.log(`\n${firm.name}: ${bodies.length} JSON bodies seen, saved top ${saved.length}. Cost: ~$0.04 (1 session).`);
})();