`scripts/refresh-listings-multi.js`

113 lines
// refresh-listings-multi.js — generalize the SFV Crexi sweep to ALL 21 CRCP metros.
// Reads data/metro-listings-config.json (per-metro cities + price band), sweeps each
// metro's city/type pages in ONE Browserbase session, captures Crexi's api.crexi.com
// asset JSON, dedupes by address, TAGS each listing with its `metro` (feeds the
// email-alerts metro filter), and APPENDS to data/listings.json.
//
// ⚠ PAID: uses Browserbase (metered cloud browser, solveCaptchas on). Gated.
//   Estimate cost before running: ~1 session, ~(sum of cities × types) page navs.
//
// Usage:
//   node scripts/refresh-listings-multi.js                 # ALL metros (paid, gated)
//   node scripts/refresh-listings-multi.js los-angeles     # one metro
//   CC_LOCAL=1 node scripts/refresh-listings-multi.js aspen  # try local Chrome ($0, may hit anti-bot)
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');

const ROOT = path.join(__dirname, '..');
const CFG = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'metro-listings-config.json'), 'utf8'));
const ONLY = process.argv[2]; // optional single metro slug
const LOCAL = process.env.CC_LOCAL === '1';
const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9 ]/g, '').replace(/\b(ave|avenue|st|street|blvd|boulevard|dr|drive|rd|road|ln|lane|pl|place|ct|court)\b/g, '').replace(/\s+/g, ' ').trim();

async function makeBrowser() {
  if (LOCAL) {
    // $0 local Chrome. Crexi is anti-bot-walled, so this may capture little — probe only.
    const execPath = process.env.CHROME_PATH || '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
    const browser = await chromium.launch({ headless: true, executablePath: execPath });
    return { browser, page: await (await browser.newContext({ viewport: { width: 1440, height: 1000 } })).newPage(), local: true };
  }
  const Browserbase = require('@browserbasehq/sdk').default;
  const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
  const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
  const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
  const session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
  console.log('bb session', session.id);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0];
  return { browser, page: ctx.pages()[0] || await ctx.newPage(), local: false };
}

(async () => {
  const metros = ONLY ? { [ONLY]: CFG.metros[ONLY] } : CFG.metros;
  if (ONLY && !CFG.metros[ONLY]) { console.error(`unknown metro: ${ONLY}`); process.exit(1); }
  const assets = new Map(); // id -> {..., metro}
  let browser;
  try {
    const b = await makeBrowser(); browser = b.browser; const page = b.page;
    page.setDefaultTimeout(45000);
    page.on('response', async (resp) => {
      if (!resp.url().includes('api.crexi.com')) return;
      try {
        const j = await resp.json();
        const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || []);
        if (Array.isArray(arr)) for (const a of arr) {
          const id = a.id || a.assetId, price = a.askingPrice || a.price, loc = (a.locations && a.locations[0]) || {};
          if (id && price && loc.city && !assets.has(id)) assets.set(id, {
            id, price: +price, city: loc.city, address: loc.address || (a.name || ''),
            cap: typeof a.capRate === 'number' ? a.capRate : null, urlSlug: a.urlSlug || '',
            desc: String(a.description || ''), types: a.types || [], metro: null,
          });
        }
      } catch (_) {}
    });

    for (const [slug, cfg] of Object.entries(metros)) {
      const cityset = new Set(cfg.cities.map(c => c.toLowerCase()));
      const before = assets.size;
      for (const [tslug] of CFG.types) {
        for (const city of cfg.cities) {
          const url = `https://www.crexi.com/properties/${cfg.state}/${city.replace(/\s+/g, '-')}/${tslug}`;
          try { await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.waitForTimeout(4500); await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
          catch (e) { console.log('nav err', slug, city, tslug, e.message.split('\n')[0]); }
        }
      }
      // tag any newly-captured, in-city, in-band asset with this metro
      for (const a of assets.values()) {
        if (a.metro) continue;
        if (cityset.has((a.city || '').toLowerCase()) && a.price >= cfg.min && a.price <= cfg.max) a.metro = slug;
      }
      console.log(`  ${slug}: session now holds ${assets.size} assets (+${assets.size - before} nav'd)`);
    }
  } catch (e) { console.error('FATAL', e.message); }
  finally { if (browser) await browser.close().catch(() => {}); }

  const cand = [...assets.values()].filter(a => a.metro && a.address);
  console.log(`captured ${assets.size} assets, ${cand.length} metro-tagged candidates`);

  const data = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'listings.json'), 'utf8'));
  const have = new Set(data.listings.map(l => norm(l.address)));
  const today = new Date().toISOString().slice(0, 10);
  let added = 0;
  for (const a of cand) {
    const key = norm(a.address);
    if (!key || have.has(key)) continue;
    have.add(key);
    const type = (a.types || []).some(t => /retail/i.test(t)) ? 'Retail' : (a.types || []).some(t => /office/i.test(t)) ? 'Office' : (a.types || []).some(t => /mixed/i.test(t)) ? 'Mixed-use' : 'Multifamily';
    const units = (() => { const m = a.desc.match(/(?:\|\s*)?\b(\d{1,3})\s*units?\b/i); if (m) { const n = +m[1]; if (n >= 1 && n <= 500 && a.price / n >= 50000) return n; } return null; })();
    data.listings.push({
      id: 'crx' + a.id, address: a.address, city: a.city, metro: a.metro, zip: '', type,
      price: a.price, units: units || 1, sqft: null, cap_rate: a.cap, verified: false,
      year_built: null, rent_control: 'verify', status: 'Active',
      upside_note: `Auto-added by multi-metro Crexi sweep ${today} (${a.metro}). ${a.cap ? 'Broker cap ' + a.cap + '% (unverified).' : 'Cap not disclosed.'} Verify before acting.`,
      source: 'https://www.crexi.com/properties/' + a.id + (a.urlSlug ? '/' + a.urlSlug : ''),
    });
    added++;
  }
  data.meta = data.meta || {};
  data.meta.multi_metro_last_run = { date: today, captured: assets.size, candidates: cand.length, added, metros: Object.keys(metros) };
  fs.writeFileSync(path.join(ROOT, 'data', 'listings.json'), JSON.stringify(data, null, 2));
  console.log(`ADDED ${added} new listings (total ${data.listings.length}).`);
})();