`scripts/enrich-missing-brokers.js`

74 lines
// enrich-missing-brokers.js — Phase 2-lite (docs/SOURCING.md): resolve the broker-of-record for the
// aggregator-only deals that don't yet have a broker block, by re-hitting CREXi /assets/<id>/brokers
// for ONLY the missing assetIds and MERGING into data/raw/broker-blocks.json (never overwrites the
// existing 1,383). One Browserbase session, batched in-page (~$0.04–0.08 total). Then re-runs
// enrich-provenance.js so ranked.json picks up the new brokers + broker-of-record deep-links.
//
//   NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/enrich-missing-brokers.js
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const SESSION_COST = 0.04;

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'ranked.json'), 'utf8')).ranked;
  const blocksPath = path.join(ROOT, 'data', 'raw', 'broker-blocks.json');
  const existing = fs.existsSync(blocksPath) ? JSON.parse(fs.readFileSync(blocksPath, 'utf8')) : [];
  const have = new Set(existing.map(x => String(x.id)));
  const missing = [...new Set(ranked
    .filter(d => /^crx\d+/.test(d.id || ''))
    .map(d => String(d.id).replace(/^crx/, ''))
    .filter(id => !have.has(id)))];

  if (!missing.length) { console.log('nothing missing — all crx deals have broker blocks.'); return; }
  process.stderr.write(`Missing broker blocks: ${missing.length} assets to fetch.\n`);

  let browser, sessions = 0;
  const collected = [];
  try {
    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
    const session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    sessions++;
    browser = await chromium.connectOverCDP(session.connectUrl);
    const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();
    page.setDefaultTimeout(45000);
    let auth = null;
    page.on('request', req => { const h = req.headers(); if (!auth && h.authorization && /bearer/i.test(h.authorization) && req.url().includes('api.crexi.com')) auth = h.authorization; });
    await page.goto('https://www.crexi.com/properties/CA/Los-Angeles/multifamily', { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(6000); await page.mouse.wheel(0, 2500).catch(() => {}); await page.waitForTimeout(2500);

    const BATCH = 25;
    for (let i = 0; i < missing.length; i += BATCH) {
      const batch = missing.slice(i, i + BATCH);
      const out = await page.evaluate(async ({ batch, auth }) => {
        const r = [];
        for (const id of batch) {
          try { const resp = await fetch(`https://api.crexi.com/assets/${id}/brokers`, { headers: auth ? { authorization: auth } : {} });
            if (resp.ok) r.push({ id, brokers: await resp.json() }); else r.push({ id, err: resp.status }); }
          catch (e) { r.push({ id, err: String(e).slice(0, 50) }); }
        }
        return r;
      }, { batch, auth });
      collected.push(...out);
      process.stderr.write(`  fetched ${Math.min(i + BATCH, missing.length)}/${missing.length}\r`);
    }
  } catch (e) { process.stderr.write('\nFATAL ' + e.message + '\n'); }
  finally { if (browser) try { await browser.close(); } catch (_) {} }

  // Merge: keep every existing block, add the newly-fetched (only those with real broker arrays).
  const fresh = collected.filter(r => Array.isArray(r.brokers) && r.brokers.length);
  const merged = existing.concat(collected.map(r => ({ id: String(r.id), brokers: Array.isArray(r.brokers) ? r.brokers : [] })));
  fs.writeFileSync(blocksPath, JSON.stringify(merged, null, 2));
  process.stderr.write(`\nFetched ${collected.length} · with brokers ${fresh.length} · broker-blocks.json now ${merged.length} rows · cost $${(sessions * SESSION_COST).toFixed(2)}\n`);

  // Re-join onto ranked.json.
  try { require('child_process').execSync('node ' + path.join(__dirname, 'enrich-provenance.js'), { stdio: 'inherit' }); }
  catch (e) { console.error('re-join failed:', e.message); }
  console.log(JSON.stringify({ fetched: collected.length, withBrokers: fresh.length, blocksTotal: merged.length, cost: '$' + (sessions * SESSION_COST).toFixed(2) }));
})();