`crawl/bb-crawl.js`

167 lines
'use strict';
// Metered Browserbase crawl for the anti-bot firms (CBRE, NAI Capital).
// Strategy: real cloud Chrome runs the site's JS, we INTERCEPT the XHR JSON the
// search app fires (its own listings API) and parse that — plus a DOM-card fallback.
// Hard per-firm time cap keeps spend bounded. Cost logged at $0.15/session-minute.
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default || require('@browserbasehq/sdk');
const base = require('./base-crawler');

const envTxt = fs.readFileSync(path.join(process.env.HOME, '.claude/skills/browserbase/.env'), 'utf8');
const API_KEY = (envTxt.match(/BROWSERBASE_API_KEY=(.*)/) || [])[1]?.trim();
const PROJECT = (envTxt.match(/BROWSERBASE_PROJECT_ID=(.*)/) || [])[1]?.trim();
const RATE = 0.15; // $/session-minute

const FIRMS = {
  cbre: { cap_ms: 6 * 60000, entries: [
    'https://www.cbre.com/properties/properties-for-lease?sort=Featured&location=Los%20Angeles,%20CA',
    'https://www.cbre.com/properties/properties-for-lease' ] },
  naicapital: { cap_ms: 4 * 60000, entries: [
    'https://www.naicapital.com/property-search/',
    'https://www.naicapital.com/listings/',
    'https://www.naicapital.com/' ] },
};

const CONSENT_URL = /onetrust|cookielaw|consent|optanon|cookiepedia|geolocation\.onetrust/i;
const CONSENT_KEYS = /groupname|optanon|cookies|isthirdparty|hostid|firstpartycookie/i;
const ADDR_KEYS = ['streetaddress', 'address1', 'address', 'street', 'propertyaddress', 'displayaddress'];
const SIZE_KEYS = ['sqft', 'squarefeet', 'rentablearea', 'size', 'buildingsize', 'availablespace', 'minsize', 'maxsize'];
const PRICE_KEYS = ['askingrent', 'rent', 'price', 'rate', 'pricepersqft'];

// A candidate is a real listing only if it has an address AND (size OR price OR geo) — and is NOT a consent object.
function isListing(o) {
  if (!o || typeof o !== 'object') return false;
  const keys = Object.keys(o).join(',').toLowerCase();
  if (CONSENT_KEYS.test(keys)) return false;
  const hasAddr = ADDR_KEYS.some(k => keys.includes(k));
  const hasSize = SIZE_KEYS.some(k => keys.includes(k));
  const hasPrice = PRICE_KEYS.some(k => keys.includes(k));
  const hasGeo = /latitude|longitude/.test(keys);
  return hasAddr && (hasSize || hasPrice || hasGeo);
}
function harvest(json, out, depth = 0) {
  if (!json || depth > 7) return;
  if (Array.isArray(json)) {
    if (json.length && json.filter(x => isListing(x)).length >= Math.min(2, json.length)) {
      for (const o of json) if (isListing(o)) out.push(o);
    } else json.forEach(x => harvest(x, out, depth + 1));
  } else if (typeof json === 'object') {
    for (const k of Object.keys(json)) harvest(json[k], out, depth + 1);
  }
}
const pick = (o, keys) => { for (const k of Object.keys(o)) if (keys.some(x => k.toLowerCase() === x || k.toLowerCase().includes(x))) { const v = o[k]; if (v != null && v !== '') return v; } return null; };
const num = (v) => { if (v == null || typeof v === 'object') return null; const n = String(v).replace(/[^0-9.]/g, ''); return n ? Math.round(+n) : null; };
// address may be a nested object {street/line1, city, ...} — flatten it to a string
function addrStr(v) {
  if (!v) return null;
  if (typeof v === 'string') return v.slice(0, 200);
  if (typeof v === 'object') return [v.street || v.line1 || v.streetAddress, v.city || v.locality].filter(Boolean).join(', ').slice(0, 200) || null;
  return null;
}

function toListing(o, firm) {
  if (!isListing(o)) return null;
  const addr = addrStr(pick(o, ADDR_KEYS));
  const title = pick(o, ['propertyname', 'name', 'title', 'buildingname']);
  const url = pick(o, ['detailurl', 'url', 'permalink', 'link', 'propertyurl', 'canonicalurl']);
  const lat = pick(o, ['latitude', 'lat']), lng = pick(o, ['longitude', 'lng', 'lon']);
  const id = pick(o, ['listingid', 'propertyid', 'id', 'slug']) || (addr ? addr.slice(0, 40) : null);
  if (!addr && !id) return null;
  const cityV = pick(o, ['city', 'cityname', 'locality']);
  return {
    external_id: String(id).slice(0, 120),
    title: String(title || addr || 'Sublease space').slice(0, 200),
    address: addr,
    city: typeof cityV === 'string' ? cityV : (typeof pick(o, ADDR_KEYS) === 'object' ? (pick(o, ADDR_KEYS).city || null) : null),
    state: (() => { const s = pick(o, ['state', 'region']); return typeof s === 'string' ? s : 'CA'; })(),
    lat: lat ? +lat : null, lng: lng ? +lng : null,
    space_type: (String(pick(o, ['propertytype', 'type', 'assetclass']) || 'office').toLowerCase().match(/office|retail|industrial|flex|medical|land/) || ['office'])[0],
    size_sf: num(pick(o, SIZE_KEYS)),
    price_amount: num(pick(o, PRICE_KEYS)),
    price_unit: (() => { const u = pick(o, ['rentunit', 'priceunit']); return typeof u === 'string' ? u : 'call for terms'; })(),
    source_url: url ? (String(url).startsWith('http') ? url : `https://www.${firm}.com${String(url).replace(/^\/?/, '/')}`) : null,
    raw: o,
  };
}

async function crawlFirm(slug) {
  const cfg = FIRMS[slug];
  const sponsor = await base.getSponsor(slug);
  if (!sponsor) { console.error(`bb-crawl: no sponsor row for ${slug}`); return { cost: 0, found: 0, added: 0 }; }
  const runId = await base.startRun(sponsor.id);
  const t0 = Date.now();
  const captured = [];
  let session, browser, found = 0, added = 0, note = '';
  try {
    const bb = new Browserbase({ apiKey: API_KEY });
    session = await bb.sessions.create({ projectId: PROJECT });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0]; const page = ctx.pages()[0] || await ctx.newPage();
    page.on('response', async (res) => {
      try {
        const ct = res.headers()['content-type'] || '';
        if (!/json/.test(ct)) return;
        const url = res.url();
        if (/analytics|telemetry|gtm|segment|sentry/i.test(url) || CONSENT_URL.test(url)) return;
        const body = await res.json().catch(() => null);
        if (body) { const arr = []; harvest(body, arr); if (arr.length) captured.push({ url, arr }); }
      } catch { /* ignore */ }
    });
    const listingsSoFar = () => captured.reduce((n, c) => n + c.arr.length, 0);
    // poll: scroll + wait, up to `budget` ms, until we have enough real listings
    const poll = async (budget) => {
      const end = Math.min(Date.now() + budget, t0 + cfg.cap_ms);
      while (Date.now() < end) {
        await page.mouse.wheel(0, 3000).catch(() => {});
        await page.waitForTimeout(3500);
        if (listingsSoFar() >= 5) return true;
      }
      return listingsSoFar() > 0;
    };
    for (const entry of cfg.entries) {
      if (Date.now() - t0 > cfg.cap_ms) break;
      try {
        await page.goto(entry, { waitUntil: 'domcontentloaded', timeout: 45000 });
        await page.waitForTimeout(3000);
        // NAI: the real search is behind a JS nav link — click into it from the homepage
        if (slug === 'naicapital' && !listingsSoFar()) {
          const link = await page.$('a[href*="propert"], a[href*="listing"], a[href*="for-lease"], a[href*="search"]');
          if (link) { await link.click().catch(() => {}); await page.waitForTimeout(3500); }
        }
        await poll(40000);
        if (listingsSoFar() >= 5) break;
      } catch (e) { note += `[${entry.split('/').pop()}:${String(e).slice(0, 40)}] `; }
    }
    // Dedupe + convert captured objects → listings
    const seen = new Set();
    const objs = captured.sort((a, b) => b.arr.length - a.arr.length).flatMap(c => c.arr);
    for (const o of objs) {
      if (Date.now() - t0 > cfg.cap_ms + 30000) break;
      const L = toListing(o, slug); if (!L) continue;
      const key = L.external_id + '|' + (L.address || '');
      if (seen.has(key)) continue; seen.add(key);
      if (!L.lat && (L.address || L.city)) { const g = await base.geocodeCensus([L.address, L.city, L.state].filter(Boolean).join(', ')); if (g) { L.lat = g.lat; L.lng = g.lng; } }
      found++; if (await base.upsertListing(sponsor.id, L)) added++;
    }
    note = `captured ${captured.length} JSON payloads (${objs.length} candidate rows). ` + note;
  } catch (e) { note += 'ERR:' + String(e).slice(0, 200); }
  finally { try { await browser?.close(); } catch {} }
  const mins = (Date.now() - t0) / 60000;
  const cost = +(mins * RATE).toFixed(2);
  await base.finishRun(runId, { status: found ? 'ok' : 'blocked', found, added, cost, notes: note.slice(0, 400) });
  console.log(`${slug}: ${found} found, ${added} new · ${mins.toFixed(1)} min · $${cost.toFixed(2)}  ${note.slice(0, 120)}`);
  return { cost, found, added };
}

(async () => {
  if (!API_KEY || !PROJECT) { console.log('missing Browserbase creds'); process.exit(1); }
  const targets = process.argv.slice(2).filter(a => FIRMS[a]);
  const run = targets.length ? targets : Object.keys(FIRMS);
  let total = 0;
  for (const slug of run) { const r = await crawlFirm(slug); total += r.cost; }
  console.log(`\n=== TOTAL BROWSERBASE SPEND: $${total.toFixed(2)} ===`);
  await base.pool.end();
})();