`scripts/probe-redfin-csv.js`

38 lines
// probe-redfin-csv.js — fetch the gis-csv feed (clean CSV) for LA condos, dump the header + 2 rows.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
const REGION = process.env.CC_REGION || '11203';

(async () => {
  let browser, session, out = {};
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);
    await page.goto(`https://www.redfin.com/city/${REGION}/CA/Los-Angeles/filter/property-type=condo`, { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(3500);
    const u = `https://www.redfin.com/stingray/api/gis-csv?al=1&region_id=${REGION}&region_type=6&uipt=3&num_homes=350&status=9&v=8`;
    const res = await page.evaluate(async (u) => {
      const r = await fetch(u, { headers: { accept: 'text/csv' } });
      const t = await r.text();
      return { status: r.status, len: t.length, sample: t.slice(0, 4000), lineCount: t.split('\n').length };
    }, u);
    out = res;
  } catch (e) { out.err = e.message; } finally { if (browser) try { await browser.close(); } catch (_) {} }
  fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-csv-probe.json'), JSON.stringify(out, null, 2));
  const lines = (out.sample || '').split('\n');
  console.log('status', out.status, 'len', out.len, 'lines', out.lineCount);
  console.log('HEADER:', lines[0]);
  console.log('ROW1:', lines[1]);
  console.log('ROW2:', lines[2]);
})();