`scripts/refresh-listings.js`

87 lines
// refresh-listings.js — weekly Browserbase/Crexi sweep that APPENDS new in-band active SFV listings
// to data/listings.json (deduped). ONE Browserbase session navigates all city/type pages and captures
// Crexi's own authed api.crexi.com asset JSON. No API key needed. Run with NODE_PATH pointing at the
// browserbase skill node_modules. Idempotent: only adds listings whose address isn't already present.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;

const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();

const NAV_CITIES = (process.env.CC_NAV ? process.env.CC_NAV.split(',').map(s => s.trim()) : ['Van Nuys', 'Reseda', 'North Hollywood', 'Northridge', 'Panorama City', 'Canoga Park', 'Sherman Oaks', 'Sun Valley', 'Encino', 'Granada Hills']);
const TYPES = [['multifamily', 'Multifamily'], ['retail', 'Retail']];
// Accept captured assets whose city is anywhere in the SFV:
const SFV = new Set(['van nuys', 'reseda', 'north hollywood', 'northridge', 'panorama city', 'canoga park',
  'sherman oaks', 'sun valley', 'encino', 'granada hills', 'tarzana', 'woodland hills', 'winnetka', 'west hills',
  'lake balboa', 'valley village', 'valley glen', 'studio city', 'toluca lake', 'arleta', 'pacoima', 'mission hills',
  'sylmar', 'tujunga', 'sunland', 'porter ranch', 'chatsworth', 'north hills'].map(s => s));
const MINP = 800000, MAXP = 1750000;
const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9 ]/g, '').replace(/\b(ave|avenue|st|street|blvd|boulevard|dr|drive|rd|road|ln|lane|pl|place|ct|court)\b/g, '').replace(/\s+/g, ' ').trim();

(async () => {
  const assets = new Map();
  let browser;
  try {
    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
    const session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    console.log('bb session', session.id);
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);
    page.on('response', async (resp) => {
      if (!resp.url().includes('api.crexi.com')) return;
      try {
        const j = await resp.json();
        const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || []);
        if (Array.isArray(arr)) for (const a of arr) {
          const id = a.id || a.assetId, price = a.askingPrice || a.price, loc = (a.locations && a.locations[0]) || {};
          if (id && price && loc.city && !assets.has(id)) assets.set(id, {
            id, price: +price, city: loc.city, address: loc.address || (a.name || ''),
            cap: typeof a.capRate === 'number' ? a.capRate : null, urlSlug: a.urlSlug || '',
            desc: String(a.description || ''), types: a.types || []
          });
        }
      } catch (_) {}
    });
    for (const [slug, label] of TYPES) {
      for (const city of NAV_CITIES) {
        const url = `https://www.crexi.com/properties/CA/${city.replace(/\s+/g, '-')}/${slug}`;
        try { await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.waitForTimeout(4500); await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
        catch (e) { console.log('nav err', city, slug, e.message.split('\n')[0]); }
      }
    }
  } catch (e) { console.error('FATAL', e.message); }
  finally { if (browser) await browser.close().catch(() => {}); }

  // Filter to in-band, SFV, plausible
  const cand = [...assets.values()].filter(a => a.city && SFV.has(a.city.toLowerCase()) && a.price >= MINP && a.price <= MAXP && a.address);
  console.log(`captured ${assets.size} assets, ${cand.length} in-band SFV candidates`);

  const data = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'listings.json'), 'utf8'));
  const have = new Set(data.listings.map(l => norm(l.address)));
  const today = new Date().toISOString().slice(0, 10);
  let added = 0;
  for (const a of cand) {
    const key = norm(a.address);
    if (!key || have.has(key)) continue;
    have.add(key);
    const type = (a.types || []).some(t => /retail/i.test(t)) ? 'Retail' : (a.types || []).some(t => /mixed/i.test(t)) ? 'Mixed-use' : 'Multifamily';
    const units = (() => { const m = a.desc.match(/(?:\|\s*)?\b(\d{1,3})\s*units?\b/i); if (m) { const n = +m[1]; if (n >= 1 && n <= 500 && a.price / n >= 50000) return n; } return null; })();
    data.listings.push({
      id: 'crx' + a.id, address: a.address, city: a.city, zip: '', type,
      price: a.price, units: units || 1, sqft: null, cap_rate: a.cap, verified: false,
      year_built: null, rent_control: 'verify', status: 'Active',
      upside_note: `Auto-added by weekly Crexi refresh ${today}. ${a.cap ? 'Broker cap ' + a.cap + '% (unverified).' : 'Cap not disclosed.'} Verify status, rent roll, and financials before acting.`,
      source: 'https://www.crexi.com/properties/' + a.id + (a.urlSlug ? '/' + a.urlSlug : '')
    });
    added++;
  }
  fs.writeFileSync(path.join(ROOT, 'data', 'listings.json'), JSON.stringify(data, null, 2));
  fs.writeFileSync(path.join(ROOT, 'data', 'last-refresh.json'), JSON.stringify({ date: today, captured: assets.size, candidates: cand.length, added }, null, 2));
  console.log(`ADDED ${added} new listings (total ${data.listings.length}).`);
})();