`scripts/fetch-condos-sample.js`

139 lines
// fetch-condos-sample.js — SMALL (1-session) Browserbase proof-of-pipeline for LA condos-for-sale.
// Drives ONE cloud-browser session to Crexi's residential/condo search for a few LA cities, captures
// the listing JSON the SPA fetches (api.crexi.com), classifies each condo's warrantability via
// classify-warrantability.js, writes data/condos-sample.json, and upserts into the cre DB `condo` table.
//
// COST: 1 Browserbase session ≈ $0.04. This is the ALLOWED small sample to prove the pipeline + feed
// graphics. The FULL-SCALE residential scrape is GATED (see pending-approval memo).
//
// Usage: NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/fetch-condos-sample.js

const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const { classify, loadFhaList, PROXY_LABEL } = require('./classify-warrantability');
let brokerdb = null; try { brokerdb = require('./db/brokers-db'); } catch (_) {}

const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');

// A few LA cities likely to have condo inventory; one session visits them sequentially.
const CITIES = (process.env.CC_CITIES || 'Los-Angeles,Long-Beach,Marina-Del-Rey,Glendale').split(',');
const OUT = path.join(__dirname, '..', 'data', 'condos-sample.json');

function condoFromAsset(a) {
  const price = a.askingPrice || a.price;
  const loc = (a.locations && a.locations[0]) || {};
  const id = a.id || a.assetId;
  if (!id || !price) return null;
  const desc = (a.description || a.marketingDescription || '') + ' ' + (a.name || '');
  const broker = (a.brokers && a.brokers[0]) || {};
  return {
    id: 'crx' + id,
    address: loc.address || loc.fullAddress || (a.name || '').slice(0, 80),
    city: (loc.city || '').trim(),
    zip: (loc.zip || loc.zipCode || '').trim(),
    price: Number(price),
    hoa: a.hoaFee || a.hoa || null,
    beds: a.bedrooms || a.beds || null,
    baths: a.bathrooms || a.baths || null,
    sqft: a.squareFeet || a.buildingSize || null,
    year_built: a.yearBuilt || null,
    project_name: a.condoProjectName || a.subdivision || a.name || null,
    listing_text: desc.slice(0, 1500),
    firm_name: (a.brokerageName || (a.brokerage && a.brokerage.name) || broker.brokerageName || null),
    broker_name: broker.name || (broker.firstName ? broker.firstName + ' ' + (broker.lastName || '') : null),
    source: a.urlSlug ? 'https://www.crexi.com/properties/' + id + '/' + a.urlSlug : 'https://www.crexi.com/properties/' + id
  };
}

(async () => {
  const captured = new Map();
  let browser, session;
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);

    page.on('response', async (resp) => {
      if (!resp.url().includes('api.crexi.com')) return;
      try {
        const j = await resp.json();
        const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || []);
        if (!Array.isArray(arr)) return;
        for (const a of arr) {
          const c = condoFromAsset(a);
          if (c && !captured.has(c.id)) captured.set(c.id, c);
        }
      } catch (_) {}
    });

    for (const citySlug of CITIES) {
      const url = `https://www.crexi.com/properties/CA/${citySlug}/condo`;
      process.stderr.write(`visiting ${url}\n`);
      try {
        await page.goto(url, { waitUntil: 'domcontentloaded' });
        await page.waitForTimeout(6000); // let the SPA fire its api.crexi.com fetches
        // scroll to trigger lazy-load of more cards
        for (let i = 0; i < 3; i++) { await page.mouse.wheel(0, 3000); await page.waitForTimeout(1500); }
      } catch (e) { process.stderr.write('  nav err: ' + e.message.split('\n')[0] + '\n'); }
    }
  } catch (e) {
    process.stderr.write('Browserbase session error: ' + e.message + '\n');
  } finally {
    if (browser) try { await browser.close(); } catch (_) {}
  }

  const condos = [...captured.values()];
  const fha = loadFhaList();
  for (const c of condos) {
    const r = classify(c, fha);
    c.warrantable_status = r.status;
    c.warrant_source = r.source;
    c.warrant_signals = r.signals;
  }

  const payload = {
    meta: {
      source: 'Crexi residential/condo search via Browserbase (1-session SAMPLE)',
      cost: '~$0.04 (1 Browserbase session)',
      scope: 'PROOF-OF-PIPELINE sample only — full-scale residential scrape is gated',
      warrantability_label: PROXY_LABEL,
      fetched_at: new Date().toISOString(),
      count: condos.length
    },
    condos
  };
  fs.writeFileSync(OUT, JSON.stringify(payload, null, 2));
  process.stderr.write(`\nCaptured ${condos.length} condos -> ${OUT}\n`);

  // Upsert into the cre DB condo table (reuse the broker graph's firm linkage).
  if (brokerdb && condos.length) {
    try {
      for (const c of condos) {
        const firmId = c.firm_name ? await brokerdb.upsertFirm(c.firm_name) : null;
        await brokerdb.pool.query(
          `INSERT INTO condo(id,address,city,zip,price,hoa,beds,baths,sqft,year_built,project_name,
             listing_text,warrantable_status,warrant_source,warrant_signals,firm_id,firm_name,broker_name,source)
           VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)
           ON CONFLICT(id) DO UPDATE SET price=EXCLUDED.price, warrantable_status=EXCLUDED.warrantable_status,
             warrant_source=EXCLUDED.warrant_source, warrant_signals=EXCLUDED.warrant_signals`,
          [c.id, c.address, c.city, c.zip, c.price, c.hoa, c.beds, c.baths, c.sqft, c.year_built,
           c.project_name, c.listing_text, c.warrantable_status, c.warrant_source,
           JSON.stringify(c.warrant_signals), firmId, c.firm_name, c.broker_name, c.source]);
      }
      process.stderr.write(`Upserted ${condos.length} into cre.condo\n`);
    } catch (e) { process.stderr.write('DB upsert err: ' + e.message + '\n'); }
    finally { try { await brokerdb.pool.end(); } catch (_) {} }
  }

  // Summary line to stdout (machine-readable).
  const by = {}; condos.forEach(c => by[c.warrantable_status] = (by[c.warrantable_status] || 0) + 1);
  console.log(JSON.stringify({ count: condos.length, byStatus: by }));
})();