Swap Browserbase transport -> openclaw in refresh-all.js (TK-10745)

2aaf18e67999d234cc700b5abf39086506a0eaa3 · 2026-08-24 04:23:30 -0700 · steve@designerwallcoverings.com

- Remove @browserbasehq/sdk import and session creation (~$0.17/session, DC-IP-blocked by Crexi)
- Connect via chromium.connectOverCDP('http://localhost:18800') — openclaw local Chrome ($0)
- Add --dry-run flag (navigate + capture, skip writes to listings.json / last-refresh.json)
- Add CC_CDP env var override for the CDP endpoint
- Disconnect gracefully (goto about:blank + browser.disconnect/close) without killing openclaw
- Update cost strings: '$0 (openclaw local Chrome)' everywhere
- Add refresh-dry npm script for easy dry-run invocation
- Dry-run verified: script connects, navigates Crexi, exits clean (0 captures = need cool-down)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Files touched

M package.json
M scripts/refresh-all.js

Diff

commit 2aaf18e67999d234cc700b5abf39086506a0eaa3
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date:   Mon Aug 24 04:23:30 2026 -0700

    Swap Browserbase transport -> openclaw in refresh-all.js (TK-10745)
    
    - Remove @browserbasehq/sdk import and session creation (~$0.17/session, DC-IP-blocked by Crexi)
    - Connect via chromium.connectOverCDP('http://localhost:18800') — openclaw local Chrome ($0)
    - Add --dry-run flag (navigate + capture, skip writes to listings.json / last-refresh.json)
    - Add CC_CDP env var override for the CDP endpoint
    - Disconnect gracefully (goto about:blank + browser.disconnect/close) without killing openclaw
    - Update cost strings: '$0 (openclaw local Chrome)' everywhere
    - Add refresh-dry npm script for easy dry-run invocation
    - Dry-run verified: script connects, navigates Crexi, exits clean (0 captures = need cool-down)
    
    Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 package.json           |  1 +
 scripts/refresh-all.js | 56 ++++++++++++++++++++++++++++++--------------------
 2 files changed, 35 insertions(+), 22 deletions(-)

diff --git a/package.json b/package.json
index b1524d1..d0c5b6f 100644
--- a/package.json
+++ b/package.json
@@ -6,6 +6,7 @@
   "scripts": {
     "scrape": "node scripts/scrape-crexi.js",
     "refresh": "NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/refresh-all.js",
+    "refresh-dry": "NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/refresh-all.js --dry-run",
     "sweep": "bash scripts/run-sweep.sh",
     "discover": "NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/discover-firm.js",
     "demographics": "node scripts/fetch-demographics.js",
diff --git a/scripts/refresh-all.js b/scripts/refresh-all.js
index 5fec70e..600424c 100644
--- a/scripts/refresh-all.js
+++ b/scripts/refresh-all.js
@@ -6,22 +6,23 @@
 //   node scripts/refresh-all.js                 # all VERIFIED firms (default: crexi only)
 //   CC_FIRMS=crexi,marcusmillichap node ...      # explicit firm set (incl. needs-discovery ones)
 //   CC_NAV="Van Nuys,Reseda" CC_FIRMS=... node ... # override the city list
+//   node scripts/refresh-all.js --dry-run        # navigate + capture, skip writes
 //
 // Idempotent: only appends listings whose normalized address isn't already present (cross-firm).
-// COST: each run = ONE Browserbase session (~$0.03–0.05 incl. captcha solves), independent of how
-// many firms/cities are swept in that session. Printed at the end.
+// TRANSPORT: openclaw local Chrome via CDP (port 18800). $0 — replaces Browserbase (~$0.17/session)
+// which was being DC-IP-blocked by Crexi. Run via:
+//   NODE_PATH=$HOME/.claude/skills/browserbase/node_modules node scripts/refresh-all.js
 
 'use strict';
 const fs = require('fs');
 const path = require('path');
 const { chromium } = require('playwright-core');
-const Browserbase = require('@browserbasehq/sdk').default;
 const { TYPES, norm, today, FIRMS, byKey, toListing, inBand } = require('./sources/firms');
 const { NAV_HUBS } = require('./sources/la-county');
 
 const ROOT = path.join(__dirname, '..');
-const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
-const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const OPENCLAW_CDP = process.env.CC_CDP || 'http://localhost:18800';
+const DRYRUN = process.argv.includes('--dry-run');
 
 // Geo scope is all of LA County (filter side). For navigation we visit the submarket HUBS by
 // default (sweeping all 88 cities in one session is infeasible); override with CC_NAV.
@@ -43,15 +44,9 @@ const SELECTED = process.env.CC_FIRMS
   const captured = new Map(SELECTED.map(f => [f.key, new Map()]));
   let browser;
   try {
-    const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
-    const session = await bb.sessions.create({
-      projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'),
-      timeout: 1800,          // 30-min TTL headroom — the full multi-city sweep exceeds the ~5-min default (prior run TIMED_OUT at 5m05s → every later goto hit a dead browser). NO keepAlive: session must end on browser.close() so we bill only for actual runtime, not the full TTL.
-      proxies: true,          // match the proven solveCaptchas pattern (captcha-solve is provisioned on the proxied path)
-      browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } }
-    });
-    console.log('bb session', session.id);
-    browser = await chromium.connectOverCDP(session.connectUrl);
+    // Attach to the shared openclaw local Chrome ($0 — residential IP, not DC-blocked by Crexi)
+    console.log(`openclaw: attaching to local Chrome CDP at ${OPENCLAW_CDP}`);
+    browser = await chromium.connectOverCDP(OPENCLAW_CDP);
     const ctx = browser.contexts()[0];
     const page = ctx.pages()[0] || await ctx.newPage();
     page.setDefaultTimeout(45000);
@@ -87,7 +82,19 @@ const SELECTED = process.env.CC_FIRMS
   } catch (e) {
     console.error('FATAL', e.message);
   } finally {
-    if (browser) await browser.close().catch(() => {});
+    // Disconnect from openclaw without closing the browser (openclaw is shared / pm2-managed)
+    if (browser) {
+      const ctx = browser.contexts?.()[0];
+      const page = ctx?.pages?.()[0];
+      if (page) await page.goto('about:blank').catch(() => {});
+      // playwright-core CDP: browser.close() sends Browser.close CDP command — use disconnect
+      // if available, otherwise let GC handle it (openclaw auto-restarts via pm2 if needed)
+      if (typeof browser.disconnect === 'function') {
+        await browser.disconnect().catch(() => {});
+      } else {
+        await browser.close().catch(() => {});
+      }
+    }
   }
 
   // ── Normalize + dedup + append (shared path) ─────────────────────────────────────────────────
@@ -110,11 +117,16 @@ const SELECTED = process.env.CC_FIRMS
     perFirm[firm.key] = { captured: captured.get(firm.key).size, inBand: cand.length, added: firmAdded };
   }
 
-  fs.writeFileSync(path.join(ROOT, 'data', 'listings.json'), JSON.stringify(data, null, 2));
-  fs.writeFileSync(path.join(ROOT, 'data', 'last-refresh.json'),
-    JSON.stringify({ date: today(), firms: perFirm, totalCaptured, added, cost: '~$0.03–0.05 (1 Browserbase session)' }, null, 2));
-
-  console.log('per-firm:', JSON.stringify(perFirm));
-  console.log(`ADDED ${added} new listings (total ${data.listings.length}). Cost: ~$0.03–0.05 (1 Browserbase session).`);
-  console.log('Next: npm run analyze  (re-rank listings.json into ranked.json for the viewer).');
+  if (DRYRUN) {
+    console.log('[DRY-RUN] per-firm:', JSON.stringify(perFirm));
+    console.log(`[DRY-RUN] Would add ${added} new listings (current total ${data.listings.length}). Cost: $0 (openclaw local).`);
+    console.log('[DRY-RUN] No files written.');
+  } else {
+    fs.writeFileSync(path.join(ROOT, 'data', 'listings.json'), JSON.stringify(data, null, 2));
+    fs.writeFileSync(path.join(ROOT, 'data', 'last-refresh.json'),
+      JSON.stringify({ date: today(), firms: perFirm, totalCaptured, added, cost: '$0 (openclaw local Chrome)' }, null, 2));
+    console.log('per-firm:', JSON.stringify(perFirm));
+    console.log(`ADDED ${added} new listings (total ${data.listings.length}). Cost: $0 (openclaw local Chrome).`);
+    console.log('Next: npm run analyze  (re-rank listings.json into ranked.json for the viewer).');
+  }
 })();

← afde0f3 auto-data-snapshot: 2026-08-24T03:41:17 (2 data files) — dat · back to Commercialrealestate · auto-data-snapshot: 2026-08-24T04:45:01 (2 data files) — dat c7d01cf →