`scripts/resolve-redfin-neighborhoods.js`

82 lines
// resolve-redfin-neighborhoods.js — resolve the LA-neighborhood misses to Redfin region ids (these are
// type-1 neighborhoods, url /neighborhood/<id>/CA/Los-Angeles/<Name>). gis feed takes region_type=1.
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const { LA_NEIGHBORHOODS } = require('./sources/la-county');
const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');

// CC_HOODS=<csv> overrides; CC_ALL=1 = full LA_NEIGHBORHOODS set (title-cased, deduped on alias spellings).
const titleCase = s => s.replace(/\b\w/g, c => c.toUpperCase());
const FULL_HOODS = [...new Set(LA_NEIGHBORHOODS.map(titleCase))];
const HOODS = (process.env.CC_HOODS ? process.env.CC_HOODS.split(',')
  : (process.env.CC_ALL === '1' ? FULL_HOODS
    : 'Van Nuys,Reseda,North Hollywood,Sherman Oaks,Encino,Woodland Hills,Northridge,Canoga Park,Studio City,Echo Park,Highland Park,Koreatown,Venice,Mar Vista,Pacific Palisades,San Pedro'.split(','))).map(s => s.trim()).filter(Boolean);
const HOODS_FILE = path.join(__dirname, '..', 'data', 'redfin-neighborhoods.json');

(async () => {
  let prior = { regions: [] };
  try { prior = JSON.parse(fs.readFileSync(HOODS_FILE, 'utf8')); } catch (_) {}
  const haveIds = new Set((prior.regions || []).filter(r => r.region_id).map(r => String(r.region_id)));
  const haveQueries = new Set((prior.regions || []).filter(r => r.region_id).map(r => (r.query || '').toLowerCase()));
  const todo = HOODS.filter(h => !haveQueries.has(h.toLowerCase()));
  process.stderr.write(`resolve-hoods: ${HOODS.length} requested, ${todo.length} new (skipping ${HOODS.length - todo.length} already resolved)\n`);

  const regions = [];
  let browser, session;
  try {
    const bb = new Browserbase({ apiKey: KEY });
    session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
    browser = await chromium.connectOverCDP(session.connectUrl);
    const ctx = browser.contexts()[0];
    const page = ctx.pages()[0] || await ctx.newPage();
    page.setDefaultTimeout(45000);
    await page.goto('https://www.redfin.com/', { waitUntil: 'domcontentloaded' });
    await page.waitForTimeout(3000);

    for (const hood of todo) {
      const r = await page.evaluate(async (q) => {
        try {
          const resp = await fetch('https://www.redfin.com/stingray/do/location-autocomplete?location=' +
            encodeURIComponent(q) + '&v=2&al=1&iss=false', { headers: { accept: 'application/json' } });
          const t = await resp.text();
          const j = JSON.parse(t.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, ''));
          const sections = (j.payload && j.payload.sections) || j.sections || [];
          const rows = [];
          for (const s of sections) for (const row of (s.rows || [])) rows.push({ name: row.name, sub: row.subName, url: row.url, type: row.type });
          // Prefer a neighborhood in Los Angeles CA.
          const hoodRow = rows.find(x => /\/neighborhood\/\d+\/CA\//i.test(x.url || '') && /los angeles|, ca/i.test(x.sub || ''));
          const cityRow = rows.find(x => /\/city\/\d+\/CA\//i.test(x.url || ''));
          const pick = hoodRow || cityRow;
          if (pick) {
            const m = pick.url.match(/\/(neighborhood|city)\/(\d+)\//);
            return { name: pick.name, sub: pick.sub, region_id: m && m[2], region_type: m && m[1] === 'neighborhood' ? 1 : 6, url: pick.url };
          }
          return { miss: true, rows: rows.slice(0, 3) };
        } catch (e) { return { err: String(e).slice(0, 100) }; }
      }, hood + ', Los Angeles, CA');
      regions.push({ query: hood, ...r });
      process.stderr.write(`  ${hood} -> ${r.region_id ? r.region_id + ' (type ' + r.region_type + ')' : (r.miss ? 'MISS' : r.err)}\n`);
      await page.waitForTimeout(400);
    }
  } catch (e) { process.stderr.write('ERR ' + e.message + '\n'); }
  finally { if (browser) try { await browser.close(); } catch (_) {} }

  const merged = [...(prior.regions || [])];
  let appended = 0, miss = [];
  for (const r of regions) {
    if (!r.region_id) { miss.push(r.query); continue; }
    if (haveIds.has(String(r.region_id))) continue;
    haveIds.add(String(r.region_id)); merged.push(r); appended++;
  }
  const resolvedTotal = merged.filter(r => r.region_id).length;
  fs.writeFileSync(HOODS_FILE,
    JSON.stringify({ resolved_at: new Date().toISOString(), count: resolvedTotal, regions: merged }, null, 2));
  console.log(JSON.stringify({ requested: HOODS.length, newly_queried: todo.length,
    appended, resolved_total: resolvedTotal, missing: miss }));
})();