← back to Commercialrealestate
CRE Job A: Redfin feed-first condo scraper (gis-csv) — region resolvers (city+neighborhood region_ids) + capped sweep w/ FHA warrantability classify + cre.condo upsert; probes confirm gis-csv feed (~350 condos/region)
e64aea1553083040917736d0b30f6d1854544bee · 2026-06-28 07:08:25 -0700 · steve@designerwallcoverings.com
Files touched
A data/condos-redfin.jsonA data/redfin-neighborhoods.jsonA data/redfin-regions.jsonA scripts/fetch-condos-redfin.jsA scripts/probe-redfin-csv.jsA scripts/probe-redfin.jsA scripts/probe-redfin2.jsA scripts/probe-redfin3.jsA scripts/resolve-redfin-neighborhoods.jsA scripts/resolve-redfin-regions.js
Diff
commit e64aea1553083040917736d0b30f6d1854544bee
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date: Sun Jun 28 07:08:25 2026 -0700
CRE Job A: Redfin feed-first condo scraper (gis-csv) — region resolvers (city+neighborhood region_ids) + capped sweep w/ FHA warrantability classify + cre.condo upsert; probes confirm gis-csv feed (~350 condos/region)
---
data/condos-redfin.json | 20 +++
data/redfin-neighborhoods.json | 133 +++++++++++++++++
data/redfin-regions.json | 220 +++++++++++++++++++++++++++++
scripts/fetch-condos-redfin.js | 243 ++++++++++++++++++++++++++++++++
scripts/probe-redfin-csv.js | 37 +++++
scripts/probe-redfin.js | 73 ++++++++++
scripts/probe-redfin2.js | 89 ++++++++++++
scripts/probe-redfin3.js | 62 ++++++++
scripts/resolve-redfin-neighborhoods.js | 58 ++++++++
scripts/resolve-redfin-regions.js | 59 ++++++++
10 files changed, 994 insertions(+)
diff --git a/data/condos-redfin.json b/data/condos-redfin.json
new file mode 100644
index 0000000..35e4280
--- /dev/null
+++ b/data/condos-redfin.json
@@ -0,0 +1,20 @@
+{
+ "meta": {
+ "source": "Redfin LA County condo search via Browserbase — feed-first (/stingray/api/gis GeoJSON)",
+ "sessions_used": 1,
+ "cost": "~$0.04 (1 Browserbase sessions @ $0.04)",
+ "cap": "HARD CAP 1 sessions / $3.20 — Redfin-only; Zillow+2captcha is a separate gate",
+ "warrantability_label": "FHA/VA-approval-based proxy, NOT lender-verified Fannie/Freddie warrantability.",
+ "fetched_at": "2026-06-28T14:02:37.224Z",
+ "count": 0,
+ "perCity": [
+ {
+ "city": "Los Angeles",
+ "gained": 0,
+ "total": 0,
+ "session": 1
+ }
+ ]
+ },
+ "condos": []
+}
\ No newline at end of file
diff --git a/data/redfin-neighborhoods.json b/data/redfin-neighborhoods.json
new file mode 100644
index 0000000..c5a5657
--- /dev/null
+++ b/data/redfin-neighborhoods.json
@@ -0,0 +1,133 @@
+{
+ "resolved_at": "2026-06-28T14:07:18.399Z",
+ "regions": [
+ {
+ "query": "Van Nuys",
+ "name": "Van Nuys",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "2859",
+ "region_type": 1,
+ "url": "/neighborhood/2859/CA/Los-Angeles/Van-Nuys"
+ },
+ {
+ "query": "Reseda",
+ "name": "Reseda",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "2257",
+ "region_type": 1,
+ "url": "/neighborhood/2257/CA/Los-Angeles/Reseda"
+ },
+ {
+ "query": "North Hollywood",
+ "name": "North Hollywood",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "1905",
+ "region_type": 1,
+ "url": "/neighborhood/1905/CA/Los-Angeles/North-Hollywood"
+ },
+ {
+ "query": "Sherman Oaks",
+ "name": "Sherman Oaks",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "2440",
+ "region_type": 1,
+ "url": "/neighborhood/2440/CA/Los-Angeles/Sherman-Oaks"
+ },
+ {
+ "query": "Encino",
+ "name": "Encino",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "899",
+ "region_type": 1,
+ "url": "/neighborhood/899/CA/Los-Angeles/Encino"
+ },
+ {
+ "query": "Woodland Hills",
+ "name": "Woodland Hills",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "10848",
+ "region_type": 1,
+ "url": "/neighborhood/10848/CA/Los-Angeles/Woodland-Hills"
+ },
+ {
+ "query": "Northridge",
+ "name": "Northridge",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "481131",
+ "region_type": 1,
+ "url": "/neighborhood/481131/CA/Los-Angeles/Northridge"
+ },
+ {
+ "query": "Canoga Park",
+ "name": "Canoga Park",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "355",
+ "region_type": 1,
+ "url": "/neighborhood/355/CA/Los-Angeles/Canoga-Park"
+ },
+ {
+ "query": "Studio City",
+ "name": "Studio City",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "2629",
+ "region_type": 1,
+ "url": "/neighborhood/2629/CA/Los-Angeles/Studio-City"
+ },
+ {
+ "query": "Echo Park",
+ "name": "Echo Park",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "762867",
+ "region_type": 1,
+ "url": "/neighborhood/762867/CA/Los-Angeles/Echo-Park"
+ },
+ {
+ "query": "Highland Park",
+ "name": "Highland Park",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "1243",
+ "region_type": 1,
+ "url": "/neighborhood/1243/CA/Los-Angeles/Highland-Park"
+ },
+ {
+ "query": "Koreatown",
+ "name": "Wilshire Center-Koreatown",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "10732",
+ "region_type": 1,
+ "url": "/neighborhood/10732/CA/Los-Angeles/Wilshire-Center-Koreatown"
+ },
+ {
+ "query": "Venice",
+ "name": "Venice",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "2861",
+ "region_type": 1,
+ "url": "/neighborhood/2861/CA/Los-Angeles/Venice"
+ },
+ {
+ "query": "Mar Vista",
+ "name": "Mar Vista",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "1635",
+ "region_type": 1,
+ "url": "/neighborhood/1635/CA/Los-Angeles/Mar-Vista"
+ },
+ {
+ "query": "Pacific Palisades",
+ "name": "Pacific Palisades",
+ "sub": "Los Angeles, CA, USA",
+ "region_id": "64066",
+ "region_type": 1,
+ "url": "/neighborhood/64066/CA/Los-Angeles/Pacific-Palisades"
+ },
+ {
+ "query": "San Pedro",
+ "name": "San Pedro Hill",
+ "sub": "Rancho Palos Verdes, CA, USA",
+ "region_id": "14609",
+ "region_type": 1,
+ "url": "/neighborhood/14609/CA/Rancho-Palos-Verdes/San-Pedro-Hill"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/data/redfin-regions.json b/data/redfin-regions.json
new file mode 100644
index 0000000..c0a9e9f
--- /dev/null
+++ b/data/redfin-regions.json
@@ -0,0 +1,220 @@
+{
+ "resolved_at": "2026-06-28T14:06:30.766Z",
+ "count": 25,
+ "regions": [
+ {
+ "query": "Van Nuys",
+ "miss": true
+ },
+ {
+ "query": "Reseda",
+ "miss": true
+ },
+ {
+ "query": "North Hollywood",
+ "miss": true
+ },
+ {
+ "query": "Sherman Oaks",
+ "miss": true
+ },
+ {
+ "query": "Encino",
+ "miss": true
+ },
+ {
+ "query": "Woodland Hills",
+ "miss": true
+ },
+ {
+ "query": "Northridge",
+ "miss": true
+ },
+ {
+ "query": "Canoga Park",
+ "miss": true
+ },
+ {
+ "query": "Studio City",
+ "miss": true
+ },
+ {
+ "query": "Burbank",
+ "name": "Burbank",
+ "region_id": "2320",
+ "url": "/city/2320/CA/Burbank"
+ },
+ {
+ "query": "Glendale",
+ "name": "Glendale",
+ "region_id": "7646",
+ "url": "/city/7646/CA/Glendale"
+ },
+ {
+ "query": "Pasadena",
+ "name": "Pasadena",
+ "region_id": "14498",
+ "url": "/city/14498/CA/Pasadena"
+ },
+ {
+ "query": "Altadena",
+ "name": "Altadena",
+ "region_id": "21169",
+ "url": "/city/21169/CA/Altadena"
+ },
+ {
+ "query": "Los Angeles",
+ "name": "Los Angeles",
+ "region_id": "11203",
+ "url": "/city/11203/CA/Los-Angeles"
+ },
+ {
+ "query": "Silver Lake",
+ "name": "Silver Lakes",
+ "region_id": "34977",
+ "url": "/city/34977/CA/Silver-Lakes"
+ },
+ {
+ "query": "Echo Park",
+ "miss": true
+ },
+ {
+ "query": "Highland Park",
+ "miss": true
+ },
+ {
+ "query": "Hollywood",
+ "name": "West Hollywood",
+ "region_id": "20771",
+ "url": "/city/20771/CA/West-Hollywood"
+ },
+ {
+ "query": "West Hollywood",
+ "name": "West Hollywood",
+ "region_id": "20771",
+ "url": "/city/20771/CA/West-Hollywood"
+ },
+ {
+ "query": "Koreatown",
+ "miss": true
+ },
+ {
+ "query": "Culver City",
+ "name": "Culver City",
+ "region_id": "4557",
+ "url": "/city/4557/CA/Culver-City"
+ },
+ {
+ "query": "Santa Monica",
+ "name": "Santa Monica",
+ "region_id": "17882",
+ "url": "/city/17882/CA/Santa-Monica"
+ },
+ {
+ "query": "Venice",
+ "miss": true
+ },
+ {
+ "query": "Mar Vista",
+ "miss": true
+ },
+ {
+ "query": "Beverly Hills",
+ "name": "Beverly Hills",
+ "region_id": "1669",
+ "url": "/city/1669/CA/Beverly-Hills"
+ },
+ {
+ "query": "Brentwood",
+ "name": "Brentwood",
+ "region_id": "2112",
+ "url": "/city/2112/CA/Brentwood"
+ },
+ {
+ "query": "Pacific Palisades",
+ "miss": true
+ },
+ {
+ "query": "Malibu",
+ "name": "Malibu",
+ "region_id": "11532",
+ "url": "/city/11532/CA/Malibu"
+ },
+ {
+ "query": "Inglewood",
+ "name": "Inglewood",
+ "region_id": "9307",
+ "url": "/city/9307/CA/Inglewood"
+ },
+ {
+ "query": "Hawthorne",
+ "name": "Hawthorne",
+ "region_id": "8306",
+ "url": "/city/8306/CA/Hawthorne"
+ },
+ {
+ "query": "Gardena",
+ "name": "Gardena",
+ "region_id": "7193",
+ "url": "/city/7193/CA/Gardena"
+ },
+ {
+ "query": "Torrance",
+ "name": "Torrance",
+ "region_id": "20094",
+ "url": "/city/20094/CA/Torrance"
+ },
+ {
+ "query": "Long Beach",
+ "name": "Long Beach",
+ "region_id": "10940",
+ "url": "/city/10940/CA/Long-Beach"
+ },
+ {
+ "query": "San Pedro",
+ "miss": true
+ },
+ {
+ "query": "Downey",
+ "name": "Downey",
+ "region_id": "5092",
+ "url": "/city/5092/CA/Downey"
+ },
+ {
+ "query": "Whittier",
+ "name": "Whittier",
+ "region_id": "20869",
+ "url": "/city/20869/CA/Whittier"
+ },
+ {
+ "query": "El Monte",
+ "name": "El Monte",
+ "region_id": "5726",
+ "url": "/city/5726/CA/El-Monte"
+ },
+ {
+ "query": "Pomona",
+ "name": "Pomona",
+ "region_id": "15047",
+ "url": "/city/15047/CA/Pomona"
+ },
+ {
+ "query": "Lancaster",
+ "name": "Lancaster",
+ "region_id": "10233",
+ "url": "/city/10233/CA/Lancaster"
+ },
+ {
+ "query": "Palmdale",
+ "name": "Palmdale",
+ "region_id": "14292",
+ "url": "/city/14292/CA/Palmdale"
+ },
+ {
+ "query": "Santa Clarita",
+ "name": "Santa Clarita",
+ "region_id": "17676",
+ "url": "/city/17676/CA/Santa-Clarita"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/scripts/fetch-condos-redfin.js b/scripts/fetch-condos-redfin.js
new file mode 100644
index 0000000..c28870a
--- /dev/null
+++ b/scripts/fetch-condos-redfin.js
@@ -0,0 +1,243 @@
+// fetch-condos-redfin.js — FEED-FIRST Redfin LA County CONDO scraper (APPROVED, capped run).
+//
+// Strategy (feed-first, no DOM scraping): for each LA submarket region (resolved to a Redfin region_id
+// by resolve-redfin-regions.js + resolve-redfin-neighborhoods.js), open ONE Browserbase session, land
+// on the region's condo city/neighborhood page (sets Redfin's expected referer/cookies), then fetch
+// Redfin's own gis-csv search feed IN-PAGE:
+// /stingray/api/gis-csv?al=1®ion_id=<id>®ion_type=<6 city|1 hood>&uipt=3&num_homes=350&status=9
+// uipt=3 = condo/co-op, status=9 = active for-sale. The CSV is clean columns (address/city/zip/price/
+// beds/baths/sqft/year/HOA/url/MLS) — far more robust than intercepting the rendered map.
+//
+// Each condo is classified via classify-warrantability.js (FHA-list match primary + heuristic flags)
+// and upserted into cre.condo with an honest PROXY label.
+//
+// HARD CAP (Steve-approved): stop at MAX_SESSIONS (80) sessions OR ~$3.20 spend, whichever first.
+// Per-session + running batch total surfaced live. Redfin-ONLY — Zillow+2captcha is a SEPARATE gate.
+//
+// Usage: NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/fetch-condos-redfin.js
+// CC_MAX_SESSIONS=80 CC_MAX_COST=3.20 CC_PER_SESSION_REGIONS=1
+
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const { classify, loadFhaList, PROXY_LABEL } = require('./classify-warrantability');
+const { inLACounty } = require('./sources/la-county');
+let brokerdb = null; try { brokerdb = require('./db/brokers-db'); } catch (_) {}
+
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+
+const SESSION_COST = 0.04;
+const MAX_SESSIONS = +(process.env.CC_MAX_SESSIONS || 80);
+const MAX_COST = +(process.env.CC_MAX_COST || 3.20);
+const PER_SESSION_REGIONS = +(process.env.CC_PER_SESSION_REGIONS || 4); // batch several regions per session to stay well under cap
+const ROOT = path.join(__dirname, '..');
+const OUT = path.join(ROOT, 'data', 'condos-redfin.json');
+
+// Build the region work-list from the two resolver outputs (dedup by region_id+type).
+function loadRegions() {
+ const out = [];
+ const seen = new Set();
+ const add = (r, region_type) => {
+ if (!r.region_id) return;
+ const key = r.region_id + ':' + region_type;
+ if (seen.has(key)) return; seen.add(key);
+ out.push({ name: r.query || r.name, region_id: r.region_id, region_type, url: r.url });
+ };
+ try {
+ const cities = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'redfin-regions.json'), 'utf8')).regions || [];
+ cities.forEach(r => add(r, 6));
+ } catch (_) {}
+ try {
+ const hoods = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'redfin-neighborhoods.json'), 'utf8')).regions || [];
+ hoods.forEach(r => add(r, r.region_type || 1));
+ } catch (_) {}
+ return out;
+}
+
+// Minimal CSV parser (handles quoted fields with commas).
+function parseCSV(text) {
+ const rows = [];
+ let i = 0, field = '', row = [], inQ = false;
+ while (i < text.length) {
+ const c = text[i];
+ if (inQ) {
+ if (c === '"') { if (text[i + 1] === '"') { field += '"'; i++; } else inQ = false; }
+ else field += c;
+ } else {
+ if (c === '"') inQ = true;
+ else if (c === ',') { row.push(field); field = ''; }
+ else if (c === '\n') { row.push(field); rows.push(row); row = []; field = ''; }
+ else if (c === '\r') { /* skip */ }
+ else field += c;
+ }
+ i++;
+ }
+ if (field.length || row.length) { row.push(field); rows.push(row); }
+ return rows;
+}
+
+function condosFromCSV(text, regionName) {
+ const rows = parseCSV(text);
+ if (!rows.length) return [];
+ // Find the header row (Redfin prepends a disclaimer line for some feeds).
+ let hi = rows.findIndex(r => r.join(',').includes('PROPERTY TYPE') && r.join(',').includes('ADDRESS'));
+ if (hi < 0) return [];
+ const H = rows[hi].map(h => h.trim());
+ const col = name => H.findIndex(h => h.toUpperCase() === name || h.toUpperCase().startsWith(name));
+ const ci = {
+ type: col('PROPERTY TYPE'), addr: col('ADDRESS'), city: col('CITY'), zip: col('ZIP OR POSTAL CODE'),
+ price: col('PRICE'), beds: col('BEDS'), baths: col('BATHS'), sqft: col('SQUARE FEET'),
+ year: col('YEAR BUILT'), hoa: col('HOA/MONTH'), status: col('STATUS'),
+ url: H.findIndex(h => h.toUpperCase().startsWith('URL')), source: col('SOURCE'), mls: col('MLS#')
+ };
+ const out = [];
+ for (let r = hi + 1; r < rows.length; r++) {
+ const row = rows[r];
+ if (!row || row.length < 5) continue;
+ const ptype = (row[ci.type] || '').trim();
+ // Keep HOA-governed attached for-sale stock the FHA list covers: condos + townhouses.
+ if (!/condo|co-?op|townhouse/i.test(ptype)) continue;
+ const price = Number((row[ci.price] || '').replace(/[^\d.]/g, ''));
+ const addr = (row[ci.addr] || '').trim();
+ const city = (row[ci.city] || '').trim();
+ if (!addr || !price) continue;
+ if (city && !inLACounty(city)) continue; // geo gate
+ const url = (row[ci.url] || '').trim();
+ const mls = (row[ci.mls] || '').trim();
+ const id = 'rdf' + (mls || (addr + (row[ci.zip] || '')).replace(/\s+/g, '').toLowerCase());
+ out.push({
+ id,
+ address: addr.slice(0, 120),
+ city,
+ zip: (row[ci.zip] || '').trim(),
+ price,
+ hoa: ci.hoa >= 0 && row[ci.hoa] ? Number(String(row[ci.hoa]).replace(/[^\d.]/g, '')) || null : null,
+ beds: ci.beds >= 0 && row[ci.beds] ? Number(row[ci.beds]) || null : null,
+ baths: ci.baths >= 0 && row[ci.baths] ? Number(row[ci.baths]) || null : null,
+ sqft: ci.sqft >= 0 && row[ci.sqft] ? Math.round(Number(String(row[ci.sqft]).replace(/[^\d.]/g, ''))) || null : null,
+ year_built: ci.year >= 0 && row[ci.year] ? Number(row[ci.year]) || null : null,
+ property_type: ptype,
+ project_name: null, // CSV has no HOA/project name; classifier uses address+zip vs FHA list
+ listing_text: ptype, // minimal text; richer description not in the feed
+ firm_name: null, // listing brokerage not in gis-csv; Job B enriches separately
+ broker_name: null,
+ source: url ? (url.startsWith('http') ? url : 'https://www.redfin.com' + url) : null,
+ _region: regionName
+ });
+ }
+ return out;
+}
+
+(async () => {
+ const regions = loadRegions();
+ if (!regions.length) { console.error('No regions resolved — run resolve-redfin-regions.js first.'); process.exit(1); }
+ process.stderr.write(`Regions to sweep: ${regions.length}\n`);
+
+ const captured = new Map();
+ let sessions = 0;
+ const perRegion = [];
+
+ // Chunk regions into sessions (PER_SESSION_REGIONS each) so we use far fewer sessions than the cap.
+ for (let g = 0; g < regions.length; g += PER_SESSION_REGIONS) {
+ if (sessions >= MAX_SESSIONS) { process.stderr.write(`\n[CAP] ${MAX_SESSIONS}-session cap reached; stopping.\n`); break; }
+ if (sessions * SESSION_COST >= MAX_COST) { process.stderr.write(`\n[CAP] $${MAX_COST.toFixed(2)} cost cap reached; stopping.\n`); break; }
+ const group = regions.slice(g, g + PER_SESSION_REGIONS);
+ let browser, session;
+ sessions++;
+ const running = (sessions * SESSION_COST).toFixed(2);
+ process.stderr.write(`\n[session ${sessions}/${MAX_SESSIONS}] regions: ${group.map(r => r.name).join(', ')} (per-session $${SESSION_COST.toFixed(2)} · running batch total $${running})\n`);
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+ // Warm up Redfin once per session.
+ await page.goto('https://www.redfin.com/city/11203/CA/Los-Angeles/filter/property-type=condo', { waitUntil: 'domcontentloaded' }).catch(() => {});
+ await page.waitForTimeout(2500);
+
+ for (const reg of group) {
+ const before = captured.size;
+ const feedUrl = `https://www.redfin.com/stingray/api/gis-csv?al=1®ion_id=${reg.region_id}®ion_type=${reg.region_type}&uipt=3&num_homes=350&status=9&sf=1,2,3,5,6,7&v=8`;
+ let csv = null;
+ try {
+ const res = await page.evaluate(async (u) => {
+ const r = await fetch(u, { headers: { accept: 'text/csv' } });
+ return { status: r.status, text: await r.text() };
+ }, feedUrl);
+ if (res.status === 200 && res.text && res.text.includes('PROPERTY TYPE')) csv = res.text;
+ else process.stderr.write(` ${reg.name}: feed status ${res.status}, ${res.text ? res.text.slice(0, 60) : 'empty'}\n`);
+ } catch (e) { process.stderr.write(` ${reg.name}: feed err ${e.message.slice(0, 60)}\n`); }
+
+ if (csv) {
+ const condos = condosFromCSV(csv, reg.name);
+ for (const c of condos) if (!captured.has(c.id)) captured.set(c.id, c);
+ const gained = captured.size - before;
+ perRegion.push({ region: reg.name, region_id: reg.region_id, region_type: reg.region_type, parsed: condos.length, gained, cumulative: captured.size });
+ process.stderr.write(` ${reg.name}: parsed ${condos.length} condos (+${gained} new, cumulative ${captured.size})\n`);
+ } else {
+ perRegion.push({ region: reg.name, region_id: reg.region_id, gained: 0, error: 'no feed' });
+ }
+ await page.waitForTimeout(800);
+ }
+ } catch (e) {
+ process.stderr.write(' session err: ' + e.message.split('\n')[0] + '\n');
+ } finally {
+ if (browser) try { await browser.close(); } catch (_) {}
+ }
+ }
+
+ const condos = [...captured.values()].map(c => { delete c._region; return c; });
+ const fha = loadFhaList();
+ for (const c of condos) {
+ const r = classify(c, fha);
+ c.warrantable_status = r.status;
+ c.warrant_source = r.source;
+ c.warrant_signals = r;
+ }
+
+ const actualCost = (sessions * SESSION_COST).toFixed(2);
+ const payload = {
+ meta: {
+ source: 'Redfin LA County condo search — feed-first (gis-csv), via Browserbase',
+ sessions_used: sessions,
+ cost: `~$${actualCost} (${sessions} Browserbase sessions @ $${SESSION_COST.toFixed(2)})`,
+ cap: `HARD CAP ${MAX_SESSIONS} sessions / $${MAX_COST.toFixed(2)} — Redfin-only; Zillow+2captcha is a separate gate`,
+ warrantability_label: PROXY_LABEL,
+ fetched_at: new Date().toISOString(),
+ count: condos.length,
+ perRegion
+ },
+ condos
+ };
+ fs.writeFileSync(OUT, JSON.stringify(payload, null, 2));
+ process.stderr.write(`\nCaptured ${condos.length} condos across ${sessions} sessions -> ${OUT}\n`);
+
+ if (brokerdb && condos.length) {
+ try {
+ for (const c of condos) {
+ const firmId = c.firm_name ? await brokerdb.upsertFirm(c.firm_name) : null;
+ await brokerdb.pool.query(
+ `INSERT INTO condo(id,address,city,zip,price,hoa,beds,baths,sqft,year_built,project_name,
+ listing_text,warrantable_status,warrant_source,warrant_signals,firm_id,firm_name,broker_name,source)
+ VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19)
+ ON CONFLICT(id) DO UPDATE SET price=EXCLUDED.price, hoa=EXCLUDED.hoa,
+ warrantable_status=EXCLUDED.warrantable_status, warrant_source=EXCLUDED.warrant_source,
+ warrant_signals=EXCLUDED.warrant_signals`,
+ [c.id, c.address, c.city, c.zip, c.price, c.hoa, c.beds, c.baths, c.sqft, c.year_built,
+ c.project_name, c.listing_text, c.warrantable_status, c.warrant_source,
+ JSON.stringify(c.warrant_signals), firmId, c.firm_name, c.broker_name, c.source]);
+ }
+ process.stderr.write(`Upserted ${condos.length} into cre.condo\n`);
+ } catch (e) { process.stderr.write('DB upsert err: ' + e.message + '\n'); }
+ finally { try { await brokerdb.pool.end(); } catch (_) {} }
+ }
+
+ const by = {}; condos.forEach(c => by[c.warrantable_status] = (by[c.warrantable_status] || 0) + 1);
+ console.log(JSON.stringify({ count: condos.length, sessions, cost: '$' + actualCost, byStatus: by }));
+})();
diff --git a/scripts/probe-redfin-csv.js b/scripts/probe-redfin-csv.js
new file mode 100644
index 0000000..0c47577
--- /dev/null
+++ b/scripts/probe-redfin-csv.js
@@ -0,0 +1,37 @@
+// probe-redfin-csv.js — fetch the gis-csv feed (clean CSV) for LA condos, dump the header + 2 rows.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+const REGION = process.env.CC_REGION || '11203';
+
+(async () => {
+ let browser, session, out = {};
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+ await page.goto(`https://www.redfin.com/city/${REGION}/CA/Los-Angeles/filter/property-type=condo`, { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(3500);
+ const u = `https://www.redfin.com/stingray/api/gis-csv?al=1®ion_id=${REGION}®ion_type=6&uipt=3&num_homes=350&status=9&v=8`;
+ const res = await page.evaluate(async (u) => {
+ const r = await fetch(u, { headers: { accept: 'text/csv' } });
+ const t = await r.text();
+ return { status: r.status, len: t.length, sample: t.slice(0, 4000), lineCount: t.split('\n').length };
+ }, u);
+ out = res;
+ } catch (e) { out.err = e.message; } finally { if (browser) try { await browser.close(); } catch (_) {} }
+ fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-csv-probe.json'), JSON.stringify(out, null, 2));
+ const lines = (out.sample || '').split('\n');
+ console.log('status', out.status, 'len', out.len, 'lines', out.lineCount);
+ console.log('HEADER:', lines[0]);
+ console.log('ROW1:', lines[1]);
+ console.log('ROW2:', lines[2]);
+})();
diff --git a/scripts/probe-redfin.js b/scripts/probe-redfin.js
new file mode 100644
index 0000000..b41c8fb
--- /dev/null
+++ b/scripts/probe-redfin.js
@@ -0,0 +1,73 @@
+// probe-redfin.js — ONE-session diagnostic: see exactly what Redfin URLs fire, capture the gis feed
+// raw, dump its top-level shape + first home record so we can fix the parser. ~$0.04.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+const CITY = process.env.CC_CITY || 'Los Angeles';
+
+(async () => {
+ const seen = [];
+ const gisDumps = [];
+ let browser, session;
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+
+ page.on('response', async (resp) => {
+ const u = resp.url();
+ if (/redfin\.com\/stingray\/api/i.test(u)) {
+ seen.push(u.slice(0, 160));
+ if (/gis/i.test(u) && gisDumps.length < 3) {
+ try {
+ let body = await resp.text();
+ const raw0 = body.slice(0, 40);
+ body = body.replace(/^\)\]\}'?/, '').replace(/^\{\}&&/, '').replace(/^\)]\}',?/, '');
+ const j = JSON.parse(body);
+ const p = j.payload || j;
+ const homes = (p.homes) || (p.sections && p.sections.flatMap(s => s.rows || [])) || [];
+ gisDumps.push({
+ url: u.slice(0, 140),
+ guard: raw0,
+ topKeys: Object.keys(j),
+ payloadKeys: p && typeof p === 'object' ? Object.keys(p) : null,
+ homeCount: homes.length,
+ firstHome: homes[0] ? JSON.stringify(homes[0]).slice(0, 2000) : null
+ });
+ } catch (e) { gisDumps.push({ url: u.slice(0, 140), parseErr: e.message.slice(0, 80) }); }
+ }
+ }
+ });
+
+ const url = `https://www.redfin.com/city/search?location=${encodeURIComponent(CITY + ', CA')}`;
+ process.stderr.write('goto ' + url + '\n');
+ await page.goto(url, { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(4000);
+ process.stderr.write('resolved to: ' + page.url() + '\n');
+ // try the condo filter
+ const cur = page.url();
+ if (/redfin\.com\/city\//i.test(cur)) {
+ const filtered = cur.split('?')[0].replace(/\/filter\/.*/, '') + '/filter/property-type=condo';
+ process.stderr.write('filter goto: ' + filtered + '\n');
+ await page.goto(filtered, { waitUntil: 'domcontentloaded' }).catch(e => process.stderr.write('filter err ' + e.message.slice(0,60) + '\n'));
+ await page.waitForTimeout(4000);
+ process.stderr.write('after filter url: ' + page.url() + '\n');
+ }
+ for (let i = 0; i < 3; i++) { await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
+ } catch (e) {
+ process.stderr.write('ERR ' + e.message + '\n');
+ } finally { if (browser) try { await browser.close(); } catch (_) {} }
+
+ fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-probe.json'),
+ JSON.stringify({ seenStingray: seen, gisDumps }, null, 2));
+ process.stderr.write(`\nstingray URLs seen: ${seen.length}, gis dumps: ${gisDumps.length}\n`);
+ console.log(JSON.stringify({ seen: seen.length, gis: gisDumps.length, homeCounts: gisDumps.map(d => d.homeCount) }));
+})();
diff --git a/scripts/probe-redfin2.js b/scripts/probe-redfin2.js
new file mode 100644
index 0000000..032d0a2
--- /dev/null
+++ b/scripts/probe-redfin2.js
@@ -0,0 +1,89 @@
+// probe-redfin2.js — resolve a city via Redfin's location-autocomplete API, navigate to the real
+// city page, capture the gis feed. Dumps everything to data/raw/redfin-probe2.json. ~$0.04.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+const CITY = process.env.CC_CITY || 'Los Angeles';
+
+const strip = s => s.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, '');
+
+(async () => {
+ const out = { city: CITY };
+ let browser, session;
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+
+ const seen = [];
+ let gis = null;
+ page.on('response', async (resp) => {
+ const u = resp.url();
+ if (/redfin\.com\/stingray\/api/i.test(u)) {
+ seen.push(u.slice(0, 150));
+ if (/gis/i.test(u) && !gis) {
+ try {
+ const j = JSON.parse(strip(await resp.text()));
+ const p = j.payload || j;
+ const homes = p.homes || (p.sections && p.sections.flatMap(s => s.rows || [])) || [];
+ gis = { url: u.slice(0, 140), payloadKeys: p && Object.keys(p), homeCount: homes.length, firstHome: homes[0] || null };
+ } catch (e) { gis = { url: u.slice(0, 140), err: e.message.slice(0, 80) }; }
+ }
+ }
+ });
+
+ // 1) Land on the Redfin homepage so an authorized fetch can hit the autocomplete API in-page.
+ await page.goto('https://www.redfin.com/', { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(3000);
+
+ // 2) Resolve the city string via the location-autocomplete endpoint, in-page.
+ const ac = await page.evaluate(async (q) => {
+ try {
+ const r = await fetch('https://www.redfin.com/stingray/do/location-autocomplete?location=' +
+ encodeURIComponent(q) + '&v=2&al=1&iss=false', { headers: { accept: 'application/json' } });
+ const t = await r.text();
+ const j = JSON.parse(t.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, ''));
+ return j;
+ } catch (e) { return { err: String(e).slice(0, 120) }; }
+ }, CITY + ', CA');
+ // Extract the first city section row's url.
+ let cityUrl = null, rowSample = null;
+ try {
+ const sections = (ac.payload && ac.payload.sections) || ac.sections || [];
+ for (const s of sections) {
+ for (const row of (s.rows || [])) {
+ if (!rowSample) rowSample = row;
+ if (row.url && /\/city\//i.test(row.url)) { cityUrl = row.url; break; }
+ }
+ if (cityUrl) break;
+ }
+ } catch (_) {}
+ out.autocomplete = { keys: ac && Object.keys(ac), rowSample, cityUrl };
+
+ // 3) Navigate to the resolved city page + condo filter.
+ if (cityUrl) {
+ const base = 'https://www.redfin.com' + cityUrl.replace(/\/$/, '');
+ const filtered = base + '/filter/property-type=condo';
+ out.cityUrl = filtered;
+ await page.goto(filtered, { waitUntil: 'domcontentloaded' }).catch(e => out.navErr = e.message.slice(0,80));
+ out.resolvedUrl = page.url();
+ await page.waitForTimeout(5000);
+ for (let i = 0; i < 3; i++) { await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
+ }
+ out.seenStingray = seen;
+ out.gis = gis;
+ } catch (e) {
+ out.err = e.message;
+ } finally { if (browser) try { await browser.close(); } catch (_) {} }
+
+ fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-probe2.json'), JSON.stringify(out, null, 2));
+ console.log(JSON.stringify({ cityUrl: out.cityUrl, resolvedUrl: out.resolvedUrl, stingray: (out.seenStingray||[]).length, gisHomes: out.gis && out.gis.homeCount }));
+})();
diff --git a/scripts/probe-redfin3.js b/scripts/probe-redfin3.js
new file mode 100644
index 0000000..61f3447
--- /dev/null
+++ b/scripts/probe-redfin3.js
@@ -0,0 +1,62 @@
+// probe-redfin3.js — fetch Redfin's gis search feed DIRECTLY in-page using region_id (11203 = LA),
+// region_type=6 (city), uipt=3 (condo property type). Dumps shape + first home. ~$0.04.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+const REGION = process.env.CC_REGION || '11203';
+
+const strip = s => s.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, '');
+
+(async () => {
+ const out = { region: REGION, attempts: [] };
+ let browser, session;
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+
+ // Land on the real LA condo city page first (sets cookies / referer Redfin expects).
+ await page.goto(`https://www.redfin.com/city/${REGION}/CA/Los-Angeles/filter/property-type=condo`, { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(4000);
+
+ // Several known gis endpoint variants; uipt=3 is condo, region_type=6 is city.
+ const urls = [
+ `https://www.redfin.com/stingray/api/gis?al=1®ion_id=${REGION}®ion_type=6&uipt=3&num_homes=350&ord=redfin-recommended-asc&page_number=1&sf=1,2,3,5,6,7&status=9&v=8`,
+ `https://www.redfin.com/stingray/api/gis-csv?al=1®ion_id=${REGION}®ion_type=6&uipt=3&num_homes=350&status=9&v=8`,
+ `https://www.redfin.com/stingray/api/gis?al=1&market=socal®ion_id=${REGION}®ion_type=6&uipt=3&num_homes=350&status=9&v=8`
+ ];
+ for (const u of urls) {
+ const res = await page.evaluate(async (u) => {
+ try {
+ const r = await fetch(u, { headers: { accept: 'application/json' } });
+ const t = await r.text();
+ return { status: r.status, len: t.length, head: t.slice(0, 60), body: t.slice(0, 200000) };
+ } catch (e) { return { err: String(e).slice(0, 120) }; }
+ }, u);
+ let parsed = null;
+ if (res.body) {
+ try {
+ const j = JSON.parse(strip(res.body));
+ const p = j.payload || j;
+ const homes = p.homes || (p.sections && p.sections.flatMap(s => s.rows || [])) || [];
+ parsed = { payloadKeys: p && Object.keys(p), homeCount: homes.length, firstHome: homes[0] || null };
+ } catch (e) { parsed = { parseErr: e.message.slice(0, 80) }; }
+ }
+ out.attempts.push({ url: u.slice(0, 110), status: res.status, len: res.len, head: res.head, err: res.err, parsed });
+ if (parsed && parsed.homeCount) break;
+ }
+ } catch (e) {
+ out.err = e.message;
+ } finally { if (browser) try { await browser.close(); } catch (_) {} }
+
+ fs.writeFileSync(path.join(__dirname, '..', 'data', 'raw', 'redfin-probe3.json'), JSON.stringify(out, null, 2));
+ console.log(JSON.stringify({ attempts: out.attempts.map(a => ({ status: a.status, homes: a.parsed && a.parsed.homeCount, head: a.head })) }));
+})();
diff --git a/scripts/resolve-redfin-neighborhoods.js b/scripts/resolve-redfin-neighborhoods.js
new file mode 100644
index 0000000..79dc8f3
--- /dev/null
+++ b/scripts/resolve-redfin-neighborhoods.js
@@ -0,0 +1,58 @@
+// resolve-redfin-neighborhoods.js — resolve the LA-neighborhood misses to Redfin region ids (these are
+// type-1 neighborhoods, url /neighborhood/<id>/CA/Los-Angeles/<Name>). gis feed takes region_type=1.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+const HOODS = (process.env.CC_HOODS || 'Van Nuys,Reseda,North Hollywood,Sherman Oaks,Encino,Woodland Hills,Northridge,Canoga Park,Studio City,Echo Park,Highland Park,Koreatown,Venice,Mar Vista,Pacific Palisades,San Pedro').split(',').map(s => s.trim());
+
+(async () => {
+ const regions = [];
+ let browser, session;
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+ await page.goto('https://www.redfin.com/', { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(3000);
+
+ for (const hood of HOODS) {
+ const r = await page.evaluate(async (q) => {
+ try {
+ const resp = await fetch('https://www.redfin.com/stingray/do/location-autocomplete?location=' +
+ encodeURIComponent(q) + '&v=2&al=1&iss=false', { headers: { accept: 'application/json' } });
+ const t = await resp.text();
+ const j = JSON.parse(t.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, ''));
+ const sections = (j.payload && j.payload.sections) || j.sections || [];
+ const rows = [];
+ for (const s of sections) for (const row of (s.rows || [])) rows.push({ name: row.name, sub: row.subName, url: row.url, type: row.type });
+ // Prefer a neighborhood in Los Angeles CA.
+ const hoodRow = rows.find(x => /\/neighborhood\/\d+\/CA\//i.test(x.url || '') && /los angeles|, ca/i.test(x.sub || ''));
+ const cityRow = rows.find(x => /\/city\/\d+\/CA\//i.test(x.url || ''));
+ const pick = hoodRow || cityRow;
+ if (pick) {
+ const m = pick.url.match(/\/(neighborhood|city)\/(\d+)\//);
+ return { name: pick.name, sub: pick.sub, region_id: m && m[2], region_type: m && m[1] === 'neighborhood' ? 1 : 6, url: pick.url };
+ }
+ return { miss: true, rows: rows.slice(0, 3) };
+ } catch (e) { return { err: String(e).slice(0, 100) }; }
+ }, hood + ', Los Angeles, CA');
+ regions.push({ query: hood, ...r });
+ process.stderr.write(` ${hood} -> ${r.region_id ? r.region_id + ' (type ' + r.region_type + ')' : (r.miss ? 'MISS' : r.err)}\n`);
+ await page.waitForTimeout(400);
+ }
+ } catch (e) { process.stderr.write('ERR ' + e.message + '\n'); }
+ finally { if (browser) try { await browser.close(); } catch (_) {} }
+
+ fs.writeFileSync(path.join(__dirname, '..', 'data', 'redfin-neighborhoods.json'),
+ JSON.stringify({ resolved_at: new Date().toISOString(), regions }, null, 2));
+ console.log(JSON.stringify({ requested: HOODS.length, resolved: regions.filter(r => r.region_id).length,
+ missing: regions.filter(r => !r.region_id).map(r => r.query) }));
+})();
diff --git a/scripts/resolve-redfin-regions.js b/scripts/resolve-redfin-regions.js
new file mode 100644
index 0000000..e10a3eb
--- /dev/null
+++ b/scripts/resolve-redfin-regions.js
@@ -0,0 +1,59 @@
+// resolve-redfin-regions.js — ONE Browserbase session: resolve every NAV_HUB city to its Redfin
+// region_id + canonical /city/<id>/CA/<Name> path via the location-autocomplete API, in-page. Writes
+// data/redfin-regions.json so the scraper can hit the gis-csv feed directly (1 session per ~all cities
+// for resolution; then 1 session per city for the actual condo feed). ~$0.04 for this whole resolve.
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const { NAV_HUBS } = require('./sources/la-county');
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+const CITIES = (process.env.CC_CITIES ? process.env.CC_CITIES.split(',') : NAV_HUBS).map(s => s.trim()).filter(Boolean);
+
+(async () => {
+ const regions = [];
+ let browser, session;
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+ await page.goto('https://www.redfin.com/', { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(3000);
+
+ for (const city of CITIES) {
+ const r = await page.evaluate(async (q) => {
+ try {
+ const resp = await fetch('https://www.redfin.com/stingray/do/location-autocomplete?location=' +
+ encodeURIComponent(q) + '&v=2&al=1&iss=false', { headers: { accept: 'application/json' } });
+ const t = await resp.text();
+ const j = JSON.parse(t.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, ''));
+ const sections = (j.payload && j.payload.sections) || j.sections || [];
+ for (const s of sections) for (const row of (s.rows || [])) {
+ // type "2" = city/place. Prefer a /city/ url in CA.
+ if (row.url && /^\/city\/\d+\/CA\//i.test(row.url)) {
+ const m = row.url.match(/\/city\/(\d+)\//);
+ return { name: row.name, region_id: m && m[1], url: row.url };
+ }
+ }
+ return { miss: true };
+ } catch (e) { return { err: String(e).slice(0, 100) }; }
+ }, city + ', CA');
+ regions.push({ query: city, ...r });
+ process.stderr.write(` ${city} -> ${r.region_id || r.miss && 'MISS' || r.err}\n`);
+ await page.waitForTimeout(400);
+ }
+ } catch (e) { process.stderr.write('ERR ' + e.message + '\n'); }
+ finally { if (browser) try { await browser.close(); } catch (_) {} }
+
+ const resolved = regions.filter(r => r.region_id);
+ fs.writeFileSync(path.join(__dirname, '..', 'data', 'redfin-regions.json'),
+ JSON.stringify({ resolved_at: new Date().toISOString(), count: resolved.length, regions }, null, 2));
+ console.log(JSON.stringify({ requested: CITIES.length, resolved: resolved.length,
+ missing: regions.filter(r => !r.region_id).map(r => r.query) }));
+})();
← d9dfc39 CRE: 5-broker enrichment PoC (~$0.04) — public Crexi broker-
·
back to Commercialrealestate
·
CRE Job A: graphics reflect REAL scraped condos — /api/condo 8bfec9f →