`scripts/resolve-redfin-regions.js`

84 lines
// resolve-redfin-regions.js — ONE Browserbase session: resolve every NAV_HUB city to its Redfin
// region_id + canonical /city/<id>/CA/<Name> path via the location-autocomplete API, in-page. Writes
// data/redfin-regions.json so the scraper can hit the gis-csv feed directly (1 session per ~all cities
// for resolution; then 1 session per city for the actual condo feed). ~$0.04 for this whole resolve.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const { NAV_HUBS, INCORPORATED, UNINCORPORATED } = require('./sources/la-county');
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');

// CC_CITIES=<csv> overrides; CC_ALL=1 = full county place set (88 incorporated + unincorporated
// communities), title-cased for the autocomplete query; default = the NAV_HUBS investment set.
const titleCase = s => s.replace(/\b\w/g, c => c.toUpperCase());
const FULL_PLACES = [...new Set([...INCORPORATED, ...UNINCORPORATED].map(titleCase))];
const CITIES = (process.env.CC_CITIES ? process.env.CC_CITIES.split(',')
  : (process.env.CC_ALL === '1' ? FULL_PLACES : NAV_HUBS)).map(s => s.trim()).filter(Boolean);
const REGIONS_FILE = path.join(__dirname, '..', 'data', 'redfin-regions.json');

(async () => {
  // Load existing so we APPEND + DEDUPE by region_id rather than overwrite (cities that don't resolve
  // are logged + skipped, never faked).
  let prior = { regions: [] };
  try { prior = JSON.parse(fs.readFileSync(REGIONS_FILE, 'utf8')); } catch (_) {}
  const haveIds = new Set((prior.regions || []).filter(r => r.region_id).map(r => String(r.region_id)));
  const haveQueries = new Set((prior.regions || []).filter(r => r.region_id).map(r => (r.query || '').toLowerCase()));
  const todo = CITIES.filter(c => !haveQueries.has(c.toLowerCase()));
  process.stderr.write(`resolve-regions: ${CITIES.length} requested, ${todo.length} new (skipping ${CITIES.length - todo.length} already resolved)\n`);

  const regions = [];
  let browser, session;
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);
    await page.goto('https://www.redfin.com/', { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(3000);

    for (const city of todo) {
      const r = await page.evaluate(async (q) => {
        try {
          const resp = await fetch('https://www.redfin.com/stingray/do/location-autocomplete?location=' +
            encodeURIComponent(q) + '&v=2&al=1&iss=false', { headers: { accept: 'application/json' } });
          const t = await resp.text();
          const j = JSON.parse(t.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, ''));
          const sections = (j.payload && j.payload.sections) || j.sections || [];
          for (const s of sections) for (const row of (s.rows || [])) {
            // type "2" = city/place. Prefer a /city/ url in CA.
            if (row.url && /^\/city\/\d+\/CA\//i.test(row.url)) {
              const m = row.url.match(/\/city\/(\d+)\//);
              return { name: row.name, region_id: m && m[1], url: row.url };
            }
          }
          return { miss: true };
        } catch (e) { return { err: String(e).slice(0, 100) }; }
      }, city + ', CA');
      regions.push({ query: city, ...r });
      process.stderr.write(`  ${city} -> ${r.region_id || r.miss && 'MISS' || r.err}\n`);
      await page.waitForTimeout(400);
    }
  } catch (e) { process.stderr.write('ERR ' + e.message + '\n'); }
  finally { if (browser) try { await browser.close(); } catch (_) {} }

  // MERGE: keep all prior entries, append newly-resolved ones whose region_id we don't already have.
  const merged = [...(prior.regions || [])];
  let appended = 0, miss = [];
  for (const r of regions) {
    if (!r.region_id) { miss.push(r.query); continue; }
    if (haveIds.has(String(r.region_id))) continue;
    haveIds.add(String(r.region_id)); merged.push(r); appended++;
  }
  const resolvedTotal = merged.filter(r => r.region_id).length;
  fs.writeFileSync(REGIONS_FILE,
    JSON.stringify({ resolved_at: new Date().toISOString(), count: resolvedTotal, regions: merged }, null, 2));
  console.log(JSON.stringify({ requested: CITIES.length, newly_queried: todo.length,
    appended, resolved_total: resolvedTotal, missing: miss }));
})();