← back to Commercialrealestate
CRE: reroute condo scrapers off Browserbase to local real Chrome ($0)
6c792a9c72bde9e74290a2fba686932833a6934a · 2026-08-18 11:47:28 -0700 · Steve Abrams
TK-10655 (Steve-approved), prefer-local rule. This machine's residential IP clears
Redfin's WAF on both the gis-csv feed AND listing pages (verified 2026-08-18), so
paid Browserbase is unnecessary:
- fetch-condos-redfin.js: flip LOCAL to the DEFAULT (CC_LOCAL=0 to opt back into
paid Browserbase); lazy-load the SDK + creds so a default run needs no bb .env.
- enrich-condo-brokers.js: RETIRE the metered --browserbase branch — both --browser
and legacy --browserbase now drive local Chrome; removed the cost-ledger billing
block (nothing to bill at $0). WAF-blocked listings still degrade gracefully.
- run-condos-report.sh: enrich step now uses --browser (local) instead of --browserbase.
Verified live $0: Redfin gis-csv 200 (Burbank 16 rows); enrich --browser extracted
broker + DRE for a listing, cost-ledger delta 0. crcp-listings (refresh-listings.js)
was already rerouted 2026-08-17.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M scripts/enrich-condo-brokers.jsM scripts/fetch-condos-redfin.jsM scripts/run-condos-report.sh
Diff
commit 6c792a9c72bde9e74290a2fba686932833a6934a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Aug 18 11:47:28 2026 -0700
CRE: reroute condo scrapers off Browserbase to local real Chrome ($0)
TK-10655 (Steve-approved), prefer-local rule. This machine's residential IP clears
Redfin's WAF on both the gis-csv feed AND listing pages (verified 2026-08-18), so
paid Browserbase is unnecessary:
- fetch-condos-redfin.js: flip LOCAL to the DEFAULT (CC_LOCAL=0 to opt back into
paid Browserbase); lazy-load the SDK + creds so a default run needs no bb .env.
- enrich-condo-brokers.js: RETIRE the metered --browserbase branch — both --browser
and legacy --browserbase now drive local Chrome; removed the cost-ledger billing
block (nothing to bill at $0). WAF-blocked listings still degrade gracefully.
- run-condos-report.sh: enrich step now uses --browser (local) instead of --browserbase.
Verified live $0: Redfin gis-csv 200 (Burbank 16 rows); enrich --browser extracted
broker + DRE for a listing, cost-ledger delta 0. crcp-listings (refresh-listings.js)
was already rerouted 2026-08-17.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/enrich-condo-brokers.js | 64 ++++++++++++++---------------------------
scripts/fetch-condos-redfin.js | 32 +++++++++++++--------
scripts/run-condos-report.sh | 8 +++---
3 files changed, 46 insertions(+), 58 deletions(-)
diff --git a/scripts/enrich-condo-brokers.js b/scripts/enrich-condo-brokers.js
index 8ab47ef..9a6457d 100644
--- a/scripts/enrich-condo-brokers.js
+++ b/scripts/enrich-condo-brokers.js
@@ -18,7 +18,8 @@
* write data/condo-brokers.json (resumable cache keyed by condo id).
*
* Blocked Redfin pages (anti-bot / non-200 / no agent block) are recorded in the run summary as
- * `blocked` ids for a metered Browserbase fallback pass (see --blocked-out). Cost: $0 local.
+ * `blocked` ids. TK-10655: the metered Browserbase fallback is RETIRED — local real Chrome on this
+ * residential IP clears Redfin's WAF, so --browser / --browserbase both run at $0. Cost: $0 local.
*
* Usage: node scripts/enrich-condo-brokers.js [--limit N] [--delay MS] [--only-missing] [--ids a,b]
*/
@@ -33,7 +34,6 @@ const BROKERS = path.join(ROOT, 'data', 'condo-brokers.json');
const DRECACHE = path.join(ROOT, 'data', 'dre-bylicense-cache.json');
const LICENSED = path.join(ROOT, 'data', 'licensed-agents.json');
const DRE_BASE = 'https://www2.dre.ca.gov/PublicASP/pplinfo.asp';
-const BB_RATE_PER_MIN = 0.15; // Browserbase billing rate ($/session-minute)
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.4 Safari/605.1.15';
const arg = (k, d) => { const i = process.argv.indexOf(k); return i > -1 ? (process.argv[i + 1] ?? true) : d; };
@@ -43,11 +43,12 @@ const ONLY_MISSING = process.argv.includes('--only-missing');
// Buyer-SIDE mode (TK-10243): target SOLD listings (where a buyer's agent can exist) instead of the
// fha_expired set, to capture "both sides" of the deal. Listing side is still refreshed on the pass.
const BUYER_SIDE = process.argv.includes('--buyer-side') || process.argv.includes('--sold');
-const USE_BROWSER = process.argv.includes('--browser'); // real Chrome via Playwright ($0, but Redfin WAF blocks flagged IPs)
-const USE_BB = process.argv.includes('--browserbase'); // metered fallback: cloud stealth browser + proxies (~$0.15/session-min)
+const USE_BROWSER = process.argv.includes('--browser'); // real Chrome via Playwright ($0)
+// --browserbase is RETIRED (TK-10655): the residential IP clears Redfin's WAF locally, so this flag
+// now ALSO routes to local Chrome ($0) instead of the old metered cloud session.
+const USE_BB = process.argv.includes('--browserbase');
const ONLY_IDS = (arg('--ids', '') || '').split(',').filter(Boolean);
const GLOBAL_MODS = require('child_process').execSync('npm root -g').toString().trim();
-const BB_SKILL = path.join(process.env.HOME, '.claude', 'skills', 'browserbase');
const sleep = ms => new Promise(r => setTimeout(r, ms));
const jitter = () => DELAY + Math.floor(Math.random() * 900);
const jload = (p, d) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return d; } };
@@ -208,43 +209,34 @@ async function main() {
? !(brokers[c.id] && Array.isArray(brokers[c.id].buyer_agents) && brokers[c.id].buyer_agents.length)
: !(brokers[c.id] && brokers[c.id].broker_name));
- // Browser context. Local real-Chrome ($0) is blocked by Redfin's AWS-WAF on flagged IPs, so the
- // metered path is Browserbase: cloud stealth browser + residential proxies + solveCaptchas that
- // clears the WAF. ONE session reused for all pages (billed per session-minute, ~$0.15/min).
- let browser = null, ctx = null, page = null, bbSession = null, startedAt = Date.now();
- if (USE_BB) {
- const { chromium } = require(BB_SKILL + '/node_modules/playwright-core');
- const Browserbase = require(BB_SKILL + '/node_modules/@browserbasehq/sdk').default;
- const env = fs.readFileSync(BB_SKILL + '/.env', 'utf8');
- const PROJECT = (env.match(/BROWSERBASE_PROJECT_ID=(.*)/) || [])[1].trim();
- const KEY = (env.match(/BROWSERBASE_API_KEY=(.*)/) || [])[1].trim();
- const bb = new Browserbase({ apiKey: KEY });
- bbSession = await bb.sessions.create({ projectId: PROJECT, proxies: true, browserSettings: { solveCaptchas: true } });
- console.log(` mode: Browserbase (metered ~$${BB_RATE_PER_MIN}/min) · session ${bbSession.id}`);
- browser = await chromium.connectOverCDP(bbSession.connectUrl);
- ctx = browser.contexts()[0]; page = ctx.pages()[0] || await ctx.newPage();
- page.setDefaultTimeout(60000);
- } else if (USE_BROWSER) {
+ // Browser context. TK-10655 (prefer-local rule): the paid Browserbase fallback is RETIRED — this
+ // machine's residential IP clears Redfin's AWS-WAF on both the gis-csv feed AND the listing pages
+ // (verified 2026-08-18: listingAgents/brokerName extracted, 200, not blocked), so both --browser
+ // and the legacy --browserbase flag now drive the LOCAL real Chrome at $0. WAF-blocked listings
+ // still degrade gracefully (recorded as `blocked`, prior data preserved) — never a paid retry.
+ let browser = null, ctx = null, page = null, startedAt = Date.now();
+ if (USE_BB || USE_BROWSER) {
const { chromium } = require(GLOBAL_MODS + '/playwright');
browser = await chromium.launch({ channel: 'chrome', headless: true });
ctx = await browser.newContext({ userAgent: UA, locale: 'en-US', viewport: { width: 1280, height: 900 } });
page = await ctx.newPage();
- console.log(' mode: real Chrome (Playwright)');
+ console.log(' mode: local real Chrome (Playwright) · $0' + (USE_BB ? ' [--browserbase retired -> local]' : ''));
}
console.log(`condo-broker enrich${BUYER_SIDE ? ' [BUYER-SIDE/sold]' : ''}: ${targets.length} target(s) of ${baseSet.length} ${BUYER_SIDE ? 'sold' : 'fha_expired'} · delay ~${DELAY}ms · $0 local`);
const blocked = [];
let done = 0, gotBroker = 0, gotDre = 0;
- // Cost/abort guardrails (Codex dissent): a hot Redfin WAF polls ~25s per blocked listing, so an
- // unbounded run can blow past its cost estimate. Bail early when the WAF is clearly hot.
+ // Abort guardrails (Codex dissent): a hot Redfin WAF polls ~25s per blocked listing, so an
+ // unbounded run wastes wall-clock. Bail early when the WAF is clearly hot. (Now $0 local — these
+ // are runtime guards, not cost guards.)
const MAX_CONSEC_BLOCKS = +process.env.MAX_CONSEC_BLOCKS || 8;
- const MAX_SESSION_MIN = +process.env.MAX_SESSION_MIN || 12; // ~$1.80 hard ceiling at BB_RATE_PER_MIN/min
+ const MAX_SESSION_MIN = +process.env.MAX_SESSION_MIN || 12; // wall-clock runtime ceiling
let consecBlocks = 0, aborted = null;
for (const c of targets) {
if (done >= LIMIT) break;
const elapsedMin = (Date.now() - startedAt) / 60000;
- if (elapsedMin > MAX_SESSION_MIN) { aborted = `session cap ${MAX_SESSION_MIN}min (~$${(MAX_SESSION_MIN * BB_RATE_PER_MIN).toFixed(2)})`; break; }
- if (consecBlocks >= MAX_CONSEC_BLOCKS) { aborted = `${consecBlocks} consecutive WAF blocks — WAF is hot, stopping to save spend`; break; }
+ if (elapsedMin > MAX_SESSION_MIN) { aborted = `runtime cap ${MAX_SESSION_MIN}min`; break; }
+ if (consecBlocks >= MAX_CONSEC_BLOCKS) { aborted = `${consecBlocks} consecutive WAF blocks — WAF is hot, stopping`; break; }
done++;
const { status, html } = await fetchRedfin(c.source, page);
let b = html ? extractBroker(html) : {};
@@ -294,19 +286,7 @@ async function main() {
process.stdout.write('\n');
if (aborted) console.log(` ⚠ ABORTED EARLY: ${aborted}. Processed ${done}/${targets.length}; rerun later (WAF cooler) with --only-missing to finish. Existing data preserved.`);
if (browser) await browser.close();
- if (bbSession) {
- const mins = (Date.now() - startedAt) / 60000;
- const cost = +(mins * BB_RATE_PER_MIN).toFixed(2);
- console.log(` Browserbase session ~${mins.toFixed(1)} min → est $${cost.toFixed(2)} (@ $${BB_RATE_PER_MIN}/session-min, + proxy bandwidth)`);
- // Log to the shared cost-ledger so AbramsEgo (:9773) tracks this spend against its budget.
- try {
- const row = { ts: new Date().toISOString(), api: 'browserbase', vendor: 'Browserbase',
- app: 'commercialrealestate/enrich-condo-brokers', note: `condo broker Redfin scrape (${gotBroker} listings, ${mins.toFixed(1)} session-min)`,
- units: [{ qty: +mins.toFixed(2), unit: 'session_min', rate: BB_RATE_PER_MIN, subtotal: cost }], cost_usd: cost };
- fs.appendFileSync(path.join(process.env.HOME, '.claude', 'cost-ledger.jsonl'), JSON.stringify(row) + '\n');
- console.log(' ✔ logged to cost-ledger (AbramsEgo budget tracking)');
- } catch (e) { console.log(' (cost-ledger log skipped:', e.message, ')'); }
- }
+ // (Browserbase cost-ledger logging removed — TK-10655: enrichment is now $0 local, nothing to bill.)
// Merge into canonical condos-redfin.json (backup first; ADD-only by id).
const stamp = new Date().toISOString().slice(0, 10);
@@ -328,7 +308,7 @@ async function main() {
}
fs.writeFileSync(CONDOS, JSON.stringify(doc, null, 1));
console.log(`✔ merged broker info into ${merged} condo records (backup: condos-redfin.json.bak-${stamp})`);
- console.log(` broker found: ${gotBroker}/${targets.length} · DRE hooked: ${gotDre} · blocked(need Browserbase): ${blocked.length}`);
+ console.log(` broker found: ${gotBroker}/${targets.length} · DRE hooked: ${gotDre} · blocked(WAF, retry later): ${blocked.length}`);
if (blocked.length) console.log(` blocked ids: ${blocked.join(',')}`);
console.log(' cost: $0 (local plain-HTTP)');
}
diff --git a/scripts/fetch-condos-redfin.js b/scripts/fetch-condos-redfin.js
index 408e999..dba794d 100644
--- a/scripts/fetch-condos-redfin.js
+++ b/scripts/fetch-condos-redfin.js
@@ -11,28 +11,32 @@
// Each condo is classified via classify-warrantability.js (FHA-list match primary + heuristic flags)
// and upserted into cre.condo with an honest PROXY label.
//
-// HARD CAP (Steve-approved): stop at MAX_SESSIONS (80) sessions OR ~$3.20 spend, whichever first.
-// Per-session + running batch total surfaced live. Redfin-ONLY — Zillow+2captcha is a SEPARATE gate.
+// TRANSPORT: LOCAL Google Chrome by DEFAULT ($0, residential IP — TK-10655/prefer-local rule).
+// Set CC_LOCAL=0 to opt back into paid Browserbase (HARD CAP 80 sessions / ~$3.20, per-session +
+// running total surfaced live). Redfin-ONLY — Zillow+2captcha is a SEPARATE gate.
//
// Usage: NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/fetch-condos-redfin.js
-// CC_MAX_SESSIONS=80 CC_MAX_COST=3.20 CC_PER_SESSION_REGIONS=1
+// CC_LOCAL=0 (paid fallback) CC_MAX_SESSIONS=80 CC_MAX_COST=3.20 CC_PER_SESSION_REGIONS=1
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
-const Browserbase = require('@browserbasehq/sdk').default;
const { classify, loadFhaList, PROXY_LABEL } = require('./classify-warrantability');
const { inLACounty } = require('./sources/la-county');
let brokerdb = null; try { brokerdb = require('./db/brokers-db'); } catch (_) {}
-const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+// LOCAL is now the DEFAULT (TK-10655, prefer-local rule): drive the local Google Chrome (residential
+// IP clears Redfin's bot wall) at $0/run. Opt back into paid Browserbase ONLY with CC_LOCAL=0.
+// Same in-page gis-csv fetch; only the browser transport differs.
+const LOCAL = process.env.CC_LOCAL !== '0';
+// Browserbase creds are read lazily below, and ONLY when CC_LOCAL=0 — so a default local run needs
+// no browserbase .env present.
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
-const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
-
-// CC_LOCAL=1 launches the local Chrome (residential IP clears Redfin's bot wall) instead of a paid
-// Browserbase session — $0/run. Same in-page gis-csv fetch; only the browser transport differs.
-const LOCAL = process.env.CC_LOCAL === '1';
+function bbCreds() {
+ const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+ return { KEY: get(env, 'BROWSERBASE_API_KEY'), PROJECT: get(env, 'BROWSERBASE_PROJECT_ID') };
+}
const CHROME_PATH = process.env.CHROME_PATH || '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
const SESSION_COST = LOCAL ? 0 : 0.04;
const MAX_SESSIONS = +(process.env.CC_MAX_SESSIONS || (LOCAL ? 999 : 80));
@@ -163,6 +167,10 @@ function condosFromCSV(text, regionName) {
if (LOCAL) {
browser = await chromium.launch({ executablePath: CHROME_PATH, args: ['--disable-blink-features=AutomationControlled'] });
} else {
+ // Paid fallback, opt-in only (CC_LOCAL=0). SDK + creds loaded lazily so a default local run
+ // never touches Browserbase.
+ const Browserbase = require('@browserbasehq/sdk').default;
+ const { KEY, PROJECT } = bbCreds();
const bb = new Browserbase({ apiKey: KEY });
session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
browser = await chromium.connectOverCDP(session.connectUrl);
@@ -228,9 +236,9 @@ function condosFromCSV(text, regionName) {
}
const payload = {
meta: {
- source: 'Redfin LA County condo search — feed-first (gis-csv), via Browserbase',
+ source: `Redfin LA County condo search — feed-first (gis-csv), via ${LOCAL ? 'local Chrome ($0)' : 'Browserbase'}`,
sessions_used: sessions,
- cost: `~$${actualCost} (${sessions} Browserbase sessions @ $${SESSION_COST.toFixed(2)})`,
+ cost: LOCAL ? '$0 (local Chrome, residential IP)' : `~$${actualCost} (${sessions} Browserbase sessions @ $${SESSION_COST.toFixed(2)})`,
cap: `HARD CAP ${MAX_SESSIONS} sessions / $${MAX_COST.toFixed(2)} — Redfin-only; Zillow+2captcha is a separate gate`,
warrantability_label: PROXY_LABEL,
fetched_at: new Date().toISOString(),
diff --git a/scripts/run-condos-report.sh b/scripts/run-condos-report.sh
index 7a54c83..eff68e1 100755
--- a/scripts/run-condos-report.sh
+++ b/scripts/run-condos-report.sh
@@ -15,10 +15,10 @@ echo "===== condos run $(date) =====" >> "$LOG"
/opt/homebrew/bin/node scripts/fetch-condos-redfin.js >> "$LOG" 2>&1 || echo "fetch-condos-redfin failed (continuing on last snapshot)" >> "$LOG"
# 1.5) Enrich NEW listings with broker + firm + DRE# + phones (--only-missing: only listings without
-# broker data are scraped — usually a handful/week, so cost stays low). The DRE# is hooked into
-# the full CA DRE record. Non-fatal. Logs its Browserbase spend to the cost-ledger, which
-# AbramsEgo (:9773) tracks against budget.
-/opt/homebrew/bin/node scripts/enrich-condo-brokers.js --only-missing --browserbase --delay 900 >> "$LOG" 2>&1 || echo "enrich-condo-brokers failed (continuing; report falls back to Redfin deep links)" >> "$LOG"
+# broker data are scraped — usually a handful/week). The DRE# is hooked into the full CA DRE
+# record. Non-fatal. --browser = LOCAL real Chrome ($0, TK-10655; residential IP clears the
+# Redfin WAF — was --browserbase, now retired).
+/opt/homebrew/bin/node scripts/enrich-condo-brokers.js --only-missing --browser --delay 900 >> "$LOG" 2>&1 || echo "enrich-condo-brokers failed (continuing; report falls back to Redfin deep links)" >> "$LOG"
# 2) Build the unwarrantable-condos HTML (fha_expired only; overlays the persistent broker cache).
REPORT_DATE="$(date +%Y-%m-%d)" /opt/homebrew/bin/node scripts/condos-report.js > data/condos-report.html 2>> "$LOG"
← fd084c5 auto-data-snapshot: 2026-08-18T09:42:51 (2 data files) — dat
·
back to Commercialrealestate
·
auto-data-snapshot: 2026-08-18T11:49:58 (2 data files) — dat 74d6c6e →