`scripts/probe-redfin2.js`

90 lines
// probe-redfin2.js — resolve a city via Redfin's location-autocomplete API, navigate to the real
// city page, capture the gis feed. Dumps everything to data/raw/redfin-probe2.json. ~$0.04.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
const CITY = process.env.CC_CITY || 'Los Angeles';

const strip = s => s.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, '');

(async () => {
  const out = { city: CITY };
  let browser, session;
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);

    const seen = [];
    let gis = null;
    page.on('response', async (resp) => {
      const u = resp.url();
      if (/redfin\.com\/stingray\/api/i.test(u)) {
        seen.push(u.slice(0, 150));
        if (/gis/i.test(u) && !gis) {
          try {
            const j = JSON.parse(strip(await resp.text()));
            const p = j.payload || j;
            const homes = p.homes || (p.sections && p.sections.flatMap(s => s.rows || [])) || [];
            gis = { url: u.slice(0, 140), payloadKeys: p && Object.keys(p), homeCount: homes.length, firstHome: homes[0] || null };
          } catch (e) { gis = { url: u.slice(0, 140), err: e.message.slice(0, 80) }; }
        }
      }
    });

    // 1) Land on the Redfin homepage so an authorized fetch can hit the autocomplete API in-page.
    await page.goto('https://www.redfin.com/', { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(3000);

    // 2) Resolve the city string via the location-autocomplete endpoint, in-page.
    const ac = await page.evaluate(async (q) => {
      try {
        const r = await fetch('https://www.redfin.com/stingray/do/location-autocomplete?location=' +
          encodeURIComponent(q) + '&v=2&al=1&iss=false', { headers: { accept: 'application/json' } });
        const t = await r.text();
        const j = JSON.parse(t.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, ''));
        return j;
      } catch (e) { return { err: String(e).slice(0, 120) }; }
    }, CITY + ', CA');
    // Extract the first city section row's url.
    let cityUrl = null, rowSample = null;
    try {
      const sections = (ac.payload && ac.payload.sections) || ac.sections || [];
      for (const s of sections) {
        for (const row of (s.rows || [])) {
          if (!rowSample) rowSample = row;
          if (row.url && /\/city\//i.test(row.url)) { cityUrl = row.url; break; }
        }
        if (cityUrl) break;
      }
    } catch (_) {}
    out.autocomplete = { keys: ac && Object.keys(ac), rowSample, cityUrl };

    // 3) Navigate to the resolved city page + condo filter.
    if (cityUrl) {
      const base = 'https://www.redfin.com' + cityUrl.replace(/\/$/, '');
      const filtered = base + '/filter/property-type=condo';
      out.cityUrl = filtered;
      await page.goto(filtered, { waitUntil: 'domcontentloaded' }).catch(e => out.navErr = e.message.slice(0,80));
      out.resolvedUrl = page.url();
      await page.waitForTimeout(5000);
      for (let i = 0; i < 3; i++) { await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
    }
    out.seenStingray = seen;
    out.gis = gis;
  } catch (e) {
    out.err = e.message;
  } finally { if (browser) try { await browser.close(); } catch (_) {} }

  fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-probe2.json'), JSON.stringify(out, null, 2));
  console.log(JSON.stringify({ cityUrl: out.cityUrl, resolvedUrl: out.resolvedUrl, stingray: (out.seenStingray||[]).length, gisHomes: out.gis && out.gis.homeCount }));
})();