← back to Commercialrealestate
residential refresh: CC_LOCAL=1 mode ($0 local Chrome, no Browserbase) + 4x/day launchd
4a6fac7c759a2f1c4652220309b2c2566ed28d4f · 2026-07-06 13:24:03 -0700 · Steve Abrams
- Both Redfin scrapers gain a CC_LOCAL branch: launch local Chrome (residential IP clears
Redfin's bot wall) instead of a paid Browserbase session. Cloud path untouched (default).
- run-residential-refresh.sh: local scrape both -> export SFR snapshot -> sanity-gate ->
rsync data-only (2 JSON snapshots) to prod. No code deploy, no pm2 reload (snapshot
fallbacks readFileSync per request). $0/run.
- launchd com.steve.crcp-residential-refresh fires 06:00/12:00/18:00/00:00.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A deploy/launchd/com.steve.crcp-residential-refresh.plistM scripts/fetch-condos-redfin.jsM scripts/fetch-sfr-redfin.jsA scripts/run-residential-refresh.sh
Diff
commit 4a6fac7c759a2f1c4652220309b2c2566ed28d4f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Jul 6 13:24:03 2026 -0700
residential refresh: CC_LOCAL=1 mode ($0 local Chrome, no Browserbase) + 4x/day launchd
- Both Redfin scrapers gain a CC_LOCAL branch: launch local Chrome (residential IP clears
Redfin's bot wall) instead of a paid Browserbase session. Cloud path untouched (default).
- run-residential-refresh.sh: local scrape both -> export SFR snapshot -> sanity-gate ->
rsync data-only (2 JSON snapshots) to prod. No code deploy, no pm2 reload (snapshot
fallbacks readFileSync per request). $0/run.
- launchd com.steve.crcp-residential-refresh fires 06:00/12:00/18:00/00:00.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
.../com.steve.crcp-residential-refresh.plist | 22 +++++++++
scripts/fetch-condos-redfin.js | 24 ++++++----
scripts/fetch-sfr-redfin.js | 24 ++++++----
scripts/run-residential-refresh.sh | 55 ++++++++++++++++++++++
4 files changed, 109 insertions(+), 16 deletions(-)
diff --git a/deploy/launchd/com.steve.crcp-residential-refresh.plist b/deploy/launchd/com.steve.crcp-residential-refresh.plist
new file mode 100644
index 0000000..6375c01
--- /dev/null
+++ b/deploy/launchd/com.steve.crcp-residential-refresh.plist
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+ <key>Label</key><string>com.steve.crcp-residential-refresh</string>
+ <key>ProgramArguments</key>
+ <array>
+ <string>/bin/bash</string>
+ <string>/Users/macstudio3/Projects/commercialrealestate/scripts/run-residential-refresh.sh</string>
+ </array>
+ <key>StartCalendarInterval</key>
+ <array>
+ <dict><key>Hour</key><integer>6</integer><key>Minute</key><integer>0</integer></dict>
+ <dict><key>Hour</key><integer>12</integer><key>Minute</key><integer>0</integer></dict>
+ <dict><key>Hour</key><integer>18</integer><key>Minute</key><integer>0</integer></dict>
+ <dict><key>Hour</key><integer>0</integer><key>Minute</key><integer>0</integer></dict>
+ </array>
+ <key>StandardOutPath</key><string>/Users/macstudio3/Projects/commercialrealestate/tmp/refresh-launchd.log</string>
+ <key>StandardErrorPath</key><string>/Users/macstudio3/Projects/commercialrealestate/tmp/refresh-launchd.log</string>
+ <key>RunAtLoad</key><false/>
+</dict>
+</plist>
diff --git a/scripts/fetch-condos-redfin.js b/scripts/fetch-condos-redfin.js
index c28870a..9ee7d88 100644
--- a/scripts/fetch-condos-redfin.js
+++ b/scripts/fetch-condos-redfin.js
@@ -30,10 +30,14 @@ const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env
const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
-const SESSION_COST = 0.04;
-const MAX_SESSIONS = +(process.env.CC_MAX_SESSIONS || 80);
-const MAX_COST = +(process.env.CC_MAX_COST || 3.20);
-const PER_SESSION_REGIONS = +(process.env.CC_PER_SESSION_REGIONS || 4); // batch several regions per session to stay well under cap
+// CC_LOCAL=1 launches the local Chrome (residential IP clears Redfin's bot wall) instead of a paid
+// Browserbase session — $0/run. Same in-page gis-csv fetch; only the browser transport differs.
+const LOCAL = process.env.CC_LOCAL === '1';
+const CHROME_PATH = process.env.CHROME_PATH || '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+const SESSION_COST = LOCAL ? 0 : 0.04;
+const MAX_SESSIONS = +(process.env.CC_MAX_SESSIONS || (LOCAL ? 999 : 80));
+const MAX_COST = +(process.env.CC_MAX_COST || 3.20); // in LOCAL mode SESSION_COST=0 so this cap never trips
+const PER_SESSION_REGIONS = +(process.env.CC_PER_SESSION_REGIONS || (LOCAL ? 20 : 4)); // more regions per local Chrome launch (launches are cheap); small batches per paid session to stay under cap
const ROOT = path.join(__dirname, '..');
const OUT = path.join(ROOT, 'data', 'condos-redfin.json');
@@ -151,10 +155,14 @@ function condosFromCSV(text, regionName) {
const running = (sessions * SESSION_COST).toFixed(2);
process.stderr.write(`\n[session ${sessions}/${MAX_SESSIONS}] regions: ${group.map(r => r.name).join(', ')} (per-session $${SESSION_COST.toFixed(2)} · running batch total $${running})\n`);
try {
- const bb = new Browserbase({ apiKey: KEY });
- session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
- browser = await chromium.connectOverCDP(session.connectUrl);
- const ctx = browser.contexts()[0];
+ if (LOCAL) {
+ browser = await chromium.launch({ executablePath: CHROME_PATH, args: ['--disable-blink-features=AutomationControlled'] });
+ } else {
+ const bb = new Browserbase({ apiKey: KEY });
+ session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ }
+ const ctx = browser.contexts()[0] || await browser.newContext({ userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', viewport: { width: 1440, height: 1000 } });
const page = ctx.pages()[0] || await ctx.newPage();
page.setDefaultTimeout(45000);
// Warm up Redfin once per session.
diff --git a/scripts/fetch-sfr-redfin.js b/scripts/fetch-sfr-redfin.js
index b961752..2fb17b9 100644
--- a/scripts/fetch-sfr-redfin.js
+++ b/scripts/fetch-sfr-redfin.js
@@ -34,10 +34,14 @@ const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.r
const ROOT = path.join(__dirname, '..');
const pool = new Pool({ host: '/tmp', port: 5432, database: 'cre', user: process.env.USER || 'stevestudio2' });
-const SESSION_COST = 0.04;
-const MAX_SESSIONS = +(process.env.CC_MAX_SESSIONS || 30);
-const MAX_COST = +(process.env.CC_MAX_COST || 1.50);
-const PER_SESSION_REGIONS = +(process.env.CC_PER_SESSION_REGIONS || 10);
+// CC_LOCAL=1 launches local Chrome (residential IP clears Redfin) instead of a paid Browserbase
+// session — $0/run. Same in-page gis-csv fetch; only the browser transport differs.
+const LOCAL = process.env.CC_LOCAL === '1';
+const CHROME_PATH = process.env.CHROME_PATH || '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+const SESSION_COST = LOCAL ? 0 : 0.04;
+const MAX_SESSIONS = +(process.env.CC_MAX_SESSIONS || (LOCAL ? 999 : 30));
+const MAX_COST = +(process.env.CC_MAX_COST || 1.50); // in LOCAL mode SESSION_COST=0 so this cap never trips
+const PER_SESSION_REGIONS = +(process.env.CC_PER_SESSION_REGIONS || (LOCAL ? 20 : 10));
const MAXR = process.env.CC_MAXREGIONS != null ? +process.env.CC_MAXREGIONS : 0; // 0 = all
const NUM_HOMES = 350;
const CAP_THRESHOLD = NUM_HOMES - 5;
@@ -169,10 +173,14 @@ function loadRegions() {
process.stderr.write(`\n[session ${sessions}/${MAX_SESSIONS}] ${group.length} regions (per-session $${SESSION_COST.toFixed(2)} · running batch total $${running} / cap $${MAX_COST.toFixed(2)})\n`);
try {
- const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
- session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
- browser = await chromium.connectOverCDP(session.connectUrl);
- const ctx = browser.contexts()[0];
+ if (LOCAL) {
+ browser = await chromium.launch({ executablePath: CHROME_PATH, args: ['--disable-blink-features=AutomationControlled'] });
+ } else {
+ const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
+ session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ }
+ const ctx = browser.contexts()[0] || await browser.newContext({ userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', viewport: { width: 1440, height: 1000 } });
const page = ctx.pages()[0] || await ctx.newPage();
page.setDefaultTimeout(45000);
await page.goto('https://www.redfin.com/city/11203/CA/Los-Angeles/filter/property-type=house', { waitUntil: 'domcontentloaded' }).catch(() => {});
diff --git a/scripts/run-residential-refresh.sh b/scripts/run-residential-refresh.sh
new file mode 100755
index 0000000..454522d
--- /dev/null
+++ b/scripts/run-residential-refresh.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+# run-residential-refresh.sh — LOCAL ($0) refresh of LA County residential inventory for
+# crcp.agentabrams.com/mls.html. Runs the Redfin gis-csv scrapers via LOCAL Chrome (CC_LOCAL=1,
+# no Browserbase spend), rebuilds the SFR snapshot, and pushes DATA-ONLY (the two JSON snapshots)
+# to the Kamatera prod box. No code is deployed and no pm2 reload is needed — the /api/residential
+# and /api/condos snapshot fallbacks readFileSync the JSON on every request, so a fresh file is live
+# on the next request.
+#
+# Cost: $0 (local Chrome on a residential IP). Scheduled 4x/day (06:00/12:00/18:00/00:00) by
+# ~/Library/LaunchAgents/com.steve.crcp-residential-refresh.plist.
+set -uo pipefail
+cd "$HOME/Projects/commercialrealestate" || exit 1
+export NODE_PATH="$HOME/.claude/skills/browserbase/node_modules"
+export CC_LOCAL=1
+export CHROME_PATH="/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
+
+DEPLOY_HOST=45.61.58.125
+DEPLOY_PATH=/root/public-projects/commercialrealestate
+TS=$(date '+%Y-%m-%d %H:%M:%S')
+LOG=tmp/residential-refresh.log
+mkdir -p tmp
+echo "===== [$TS] residential refresh (LOCAL, \$0) =====" >> "$LOG"
+
+# 1) Scrape condos (full sweep, local Chrome). Upserts cre.condo + rewrites data/condos-redfin.json.
+echo "[$(date '+%H:%M:%S')] condos..." >> "$LOG"
+node scripts/fetch-condos-redfin.js >> "$LOG" 2>&1
+CONDO_RC=$?
+
+# 2) Scrape SFR (full sweep, local Chrome). Upserts cre.sfr only.
+echo "[$(date '+%H:%M:%S')] sfr..." >> "$LOG"
+node scripts/fetch-sfr-redfin.js >> "$LOG" 2>&1
+SFR_RC=$?
+
+# 3) Export the SFR snapshot from the DB (fetch-sfr writes DB only).
+echo "[$(date '+%H:%M:%S')] export sfr snapshot..." >> "$LOG"
+node scripts/export-sfr-snapshot.js >> "$LOG" 2>&1
+EXP_RC=$?
+
+# 4) Guard: never push a truncated snapshot. Require both files to be sane before rsync.
+SFR_N=$(node -e 'try{console.log(require("./data/sfr-redfin.json").sfr.length)}catch(e){console.log(0)}')
+CONDO_N=$(node -e 'try{const c=require("./data/condos-redfin.json");console.log((c.condos||c).length)}catch(e){console.log(0)}')
+echo "[$(date '+%H:%M:%S')] snapshot counts — sfr=$SFR_N condo=$CONDO_N (rc: condo=$CONDO_RC sfr=$SFR_RC exp=$EXP_RC)" >> "$LOG"
+
+if [ "$SFR_N" -lt 5000 ] || [ "$CONDO_N" -lt 500 ]; then
+ echo "[$(date '+%H:%M:%S')] ABORT push — snapshot below sanity floor (sfr<5000 or condo<500). Keeping prod as-is." >> "$LOG"
+ exit 2
+fi
+
+# 5) Push DATA-ONLY to prod (two files). No code, no reload.
+echo "[$(date '+%H:%M:%S')] rsync snapshots -> prod..." >> "$LOG"
+rsync -az data/sfr-redfin.json data/condos-redfin.json \
+ "root@${DEPLOY_HOST}:${DEPLOY_PATH}/data/" >> "$LOG" 2>&1
+RSYNC_RC=$?
+echo "[$(date '+%H:%M:%S')] DONE — rsync rc=$RSYNC_RC · cost \$0 (local) · live: https://crcp.agentabrams.com/mls.html" >> "$LOG"
+exit $RSYNC_RC
← 9a70d22 residential: fresh Redfin scrape snapshots (10.6k SFR + 1.29
·
back to Commercialrealestate
·
refresh: set CC_FRESH=1 so SFR re-sweeps every run (ledger w dcf9b4d →