`scripts/probe-redfin.js`

74 lines
// probe-redfin.js — ONE-session diagnostic: see exactly what Redfin URLs fire, capture the gis feed
// raw, dump its top-level shape + first home record so we can fix the parser. ~$0.04.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
const CITY = process.env.CC_CITY || 'Los Angeles';

(async () => {
  const seen = [];
  const gisDumps = [];
  let browser, session;
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);

    page.on('response', async (resp) => {
      const u = resp.url();
      if (/redfin\.com\/stingray\/api/i.test(u)) {
        seen.push(u.slice(0, 160));
        if (/gis/i.test(u) && gisDumps.length < 3) {
          try {
            let body = await resp.text();
            const raw0 = body.slice(0, 40);
            body = body.replace(/^\)\]\}'?/, '').replace(/^\{\}&&/, '').replace(/^\)]\}',?/, '');
            const j = JSON.parse(body);
            const p = j.payload || j;
            const homes = (p.homes) || (p.sections && p.sections.flatMap(s => s.rows || [])) || [];
            gisDumps.push({
              url: u.slice(0, 140),
              guard: raw0,
              topKeys: Object.keys(j),
              payloadKeys: p && typeof p === 'object' ? Object.keys(p) : null,
              homeCount: homes.length,
              firstHome: homes[0] ? JSON.stringify(homes[0]).slice(0, 2000) : null
            });
          } catch (e) { gisDumps.push({ url: u.slice(0, 140), parseErr: e.message.slice(0, 80) }); }
        }
      }
    });

    const url = `https://www.redfin.com/city/search?location=${encodeURIComponent(CITY + ', CA')}`;
    process.stderr.write('goto ' + url + '\n');
    await page.goto(url, { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(4000);
    process.stderr.write('resolved to: ' + page.url() + '\n');
    // try the condo filter
    const cur = page.url();
    if (/redfin\.com\/city\//i.test(cur)) {
      const filtered = cur.split('?')[0].replace(/\/filter\/.*/, '') + '/filter/property-type=condo';
      process.stderr.write('filter goto: ' + filtered + '\n');
      await page.goto(filtered, { waitUntil: 'domcontentloaded' }).catch(e => process.stderr.write('filter err ' + e.message.slice(0,60) + '\n'));
      await page.waitForTimeout(4000);
      process.stderr.write('after filter url: ' + page.url() + '\n');
    }
    for (let i = 0; i < 3; i++) { await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
  } catch (e) {
    process.stderr.write('ERR ' + e.message + '\n');
  } finally { if (browser) try { await browser.close(); } catch (_) {} }

  fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-probe.json'),
    JSON.stringify({ seenStingray: seen, gisDumps }, null, 2));
  process.stderr.write(`\nstingray URLs seen: ${seen.length}, gis dumps: ${gisDumps.length}\n`);
  console.log(JSON.stringify({ seen: seen.length, gis: gisDumps.length, homeCounts: gisDumps.map(d => d.homeCount) }));
})();