`scripts/probe-redfin3.js`

63 lines
// probe-redfin3.js — fetch Redfin's gis search feed DIRECTLY in-page using region_id (11203 = LA),
// region_type=6 (city), uipt=3 (condo property type). Dumps shape + first home. ~$0.04.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
const REGION = process.env.CC_REGION || '11203';

const strip = s => s.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, '');

(async () => {
  const out = { region: REGION, attempts: [] };
  let browser, session;
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);

    // Land on the real LA condo city page first (sets cookies / referer Redfin expects).
    await page.goto(`https://www.redfin.com/city/${REGION}/CA/Los-Angeles/filter/property-type=condo`, { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(4000);

    // Several known gis endpoint variants; uipt=3 is condo, region_type=6 is city.
    const urls = [
      `https://www.redfin.com/stingray/api/gis?al=1&region_id=${REGION}&region_type=6&uipt=3&num_homes=350&ord=redfin-recommended-asc&page_number=1&sf=1,2,3,5,6,7&status=9&v=8`,
      `https://www.redfin.com/stingray/api/gis-csv?al=1&region_id=${REGION}&region_type=6&uipt=3&num_homes=350&status=9&v=8`,
      `https://www.redfin.com/stingray/api/gis?al=1&market=socal&region_id=${REGION}&region_type=6&uipt=3&num_homes=350&status=9&v=8`
    ];
    for (const u of urls) {
      const res = await page.evaluate(async (u) => {
        try {
          const r = await fetch(u, { headers: { accept: 'application/json' } });
          const t = await r.text();
          return { status: r.status, len: t.length, head: t.slice(0, 60), body: t.slice(0, 200000) };
        } catch (e) { return { err: String(e).slice(0, 120) }; }
      }, u);
      let parsed = null;
      if (res.body) {
        try {
          const j = JSON.parse(strip(res.body));
          const p = j.payload || j;
          const homes = p.homes || (p.sections && p.sections.flatMap(s => s.rows || [])) || [];
          parsed = { payloadKeys: p && Object.keys(p), homeCount: homes.length, firstHome: homes[0] || null };
        } catch (e) { parsed = { parseErr: e.message.slice(0, 80) }; }
      }
      out.attempts.push({ url: u.slice(0, 110), status: res.status, len: res.len, head: res.head, err: res.err, parsed });
      if (parsed && parsed.homeCount) break;
    }
  } catch (e) {
    out.err = e.message;
  } finally { if (browser) try { await browser.close(); } catch (_) {} }

  fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-probe3.json'), JSON.stringify(out, null, 2));
  console.log(JSON.stringify({ attempts: out.attempts.map(a => ({ status: a.status, homes: a.parsed && a.parsed.homeCount, head: a.head })) }));
})();