`scripts/refresh-all.js`

119 lines
// refresh-all.js — multi-firm sweep. Supersedes refresh-listings.js (which was Crexi-only) by
// driving EVERY firm in sources/firms.js through one shared capture → normalize → dedup → append
// path into data/listings.json. Crexi remains the proven source; the brokerage firms (Marcus &
// Millichap, Lee & Associates, Colliers, CBRE, JLL) are included once their schema is confirmed.
//
//   node scripts/refresh-all.js                 # all VERIFIED firms (default: crexi only)
//   CC_FIRMS=crexi,marcusmillichap node ...      # explicit firm set (incl. needs-discovery ones)
//   CC_NAV="Van Nuys,Reseda" CC_FIRMS=... node ... # override the city list
//
// Idempotent: only appends listings whose normalized address isn't already present (cross-firm).
// COST: each run = ONE Browserbase session (~$0.03–0.05 incl. captcha solves), independent of how
// many firms/cities are swept in that session. Printed at the end.

'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const { TYPES, norm, today, FIRMS, byKey, toListing, inBand } = require('./sources/firms');
const { NAV_HUBS } = require('./sources/la-county');

const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();

// Geo scope is all of LA County (filter side). For navigation we visit the submarket HUBS by
// default (sweeping all 88 cities in one session is infeasible); override with CC_NAV.
const NAV_CITIES = process.env.CC_NAV
  ? process.env.CC_NAV.split(',').map(s => s.trim())
  : NAV_HUBS;

// Which firms to run. Default = verified only, so an unattended cron never silently scrapes an
// unconfirmed firm. Pass CC_FIRMS to opt specific firms in.
const SELECTED = process.env.CC_FIRMS
  ? process.env.CC_FIRMS.split(',').map(s => s.trim()).map(byKey).filter(Boolean)
  : FIRMS.filter(f => f.status === 'verified');

(async () => {
  if (!SELECTED.length) { console.error('no firms selected'); process.exit(2); }
  console.log('firms:', SELECTED.map(f => `${f.key}(${f.status})`).join(', '));

  // Per-firm bucket of captured intermediate assets, keyed by firm.key.
  const captured = new Map(SELECTED.map(f => [f.key, new Map()]));
  let browser;
  try {
    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
    const session = await bb.sessions.create({
      projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'),
      browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } }
    });
    console.log('bb session', session.id);
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);

    // ONE response listener routes each JSON body to whichever firm owns its host, then runs that
    // firm's extractor. This is the generalization of refresh-listings.js's single crexi hook.
    page.on('response', async (resp) => {
      const url = resp.url();
      const firm = SELECTED.find(f => f.apiHosts.some(h => url.includes(h)));
      if (!firm) return;
      let j; try { j = await resp.json(); } catch { return; }
      let assets; try { assets = firm.extract(j) || []; } catch { return; }
      const bucket = captured.get(firm.key);
      for (const a of assets) if (a && a.id && !bucket.has(a.id)) bucket.set(a.id, a);
    });

    // Navigate each firm's search pages (city × type) to make its XHRs fire.
    for (const firm of SELECTED) {
      for (const [slug] of TYPES) {
        for (const city of NAV_CITIES) {
          for (const url of firm.navUrls(city, slug)) {
            try {
              await page.goto(url, { waitUntil: 'domcontentloaded' });
              await page.waitForTimeout(4500);
              await page.mouse.wheel(0, 3000).catch(() => {});
              await page.waitForTimeout(1500);
            } catch (e) { console.log('nav err', firm.key, city, slug, String(e.message).split('\n')[0]); }
          }
        }
      }
      console.log(`  ${firm.key}: captured ${captured.get(firm.key).size} raw assets`);
    }
  } catch (e) {
    console.error('FATAL', e.message);
  } finally {
    if (browser) await browser.close().catch(() => {});
  }

  // ── Normalize + dedup + append (shared path) ─────────────────────────────────────────────────
  const data = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'listings.json'), 'utf8'));
  const have = new Set(data.listings.map(l => norm(l.address)));
  const perFirm = {};
  let added = 0, totalCaptured = 0;

  for (const firm of SELECTED) {
    const cand = [...captured.get(firm.key).values()].filter(inBand);
    totalCaptured += captured.get(firm.key).size;
    let firmAdded = 0;
    for (const a of cand) {
      const key = norm(a.address);
      if (!key || have.has(key)) continue;       // cross-firm dedup: first firm to list an address wins
      have.add(key);
      data.listings.push(toListing(a, firm));
      firmAdded++; added++;
    }
    perFirm[firm.key] = { captured: captured.get(firm.key).size, inBand: cand.length, added: firmAdded };
  }

  fs.writeFileSync(path.join(ROOT, 'data', 'listings.json'), JSON.stringify(data, null, 2));
  fs.writeFileSync(path.join(ROOT, 'data', 'last-refresh.json'),
    JSON.stringify({ date: today(), firms: perFirm, totalCaptured, added, cost: '~$0.03–0.05 (1 Browserbase session)' }, null, 2));

  console.log('per-firm:', JSON.stringify(perFirm));
  console.log(`ADDED ${added} new listings (total ${data.listings.length}). Cost: ~$0.03–0.05 (1 Browserbase session).`);
  console.log('Next: npm run analyze  (re-rank listings.json into ranked.json for the viewer).');
})();