← back to Commercialrealestate
Live comps -> Browserbase/Crexi (drop Exa dependency)
89f1ab2efba4b3865299492c3b34efa62eb0740b · 2026-06-21 17:46:28 -0700 · Steve
- LoopNet hard-blocks cloud browsers (Access Denied/PerimeterX) -> switched source to Crexi
- scripts/live-comps-bb.js: Browserbase cloud browser loads Crexi city/type page and intercepts
the SPA's own authed api.crexi.com JSON (geo-scoped) -> real comps (price/cap/units/address)
- serve.js /api/comps now shells to the Browserbase scraper (~$0.03/call), no API key needed
- viewer button relabeled ~$0.03; renders price/cap/units; feeds chat context
- verified live: Reseda multifamily -> 8 real comps
Files touched
M public/index.htmlA scripts/live-comps-bb.jsM scripts/serve.js
Diff
commit 89f1ab2efba4b3865299492c3b34efa62eb0740b
Author: Steve <steve@designerwallcoverings.com>
Date: Sun Jun 21 17:46:28 2026 -0700
Live comps -> Browserbase/Crexi (drop Exa dependency)
- LoopNet hard-blocks cloud browsers (Access Denied/PerimeterX) -> switched source to Crexi
- scripts/live-comps-bb.js: Browserbase cloud browser loads Crexi city/type page and intercepts
the SPA's own authed api.crexi.com JSON (geo-scoped) -> real comps (price/cap/units/address)
- serve.js /api/comps now shells to the Browserbase scraper (~$0.03/call), no API key needed
- viewer button relabeled ~$0.03; renders price/cap/units; feeds chat context
- verified live: Reseda multifamily -> 8 real comps
---
public/index.html | 12 +++----
scripts/live-comps-bb.js | 93 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/serve.js | 37 ++++++++-----------
3 files changed, 114 insertions(+), 28 deletions(-)
diff --git a/public/index.html b/public/index.html
index 651c5a6..3f0b58c 100644
--- a/public/index.html
+++ b/public/index.html
@@ -178,7 +178,7 @@ function card(p){
<div class="thesis">${p.qwen?.thesis||''}</div>
${p.qwen?.risks?.length?`<div><span class="small">Risks</span><ul class="lst">${p.qwen.risks.map(r=>`<li>${r}</li>`).join('')}</ul></div>`:''}
${p.upside_note?`<div class="small">💡 ${p.upside_note}</div>`:''}
- <div class="src"><a href="${p.source}" target="_blank" rel="noopener noreferrer">View listing ↗</a> · <button class="lcbtn" data-id="${p.id}" title="Pull fresh comparable sales from the web (costs ~$0.01)">🔄 Live comps (~$0.01)</button></div>
+ <div class="src"><a href="${p.source}" target="_blank" rel="noopener noreferrer">View listing ↗</a> · <button class="lcbtn" data-id="${p.id}" title="Pull fresh comps from LoopNet via a cloud browser (~$0.03)">🔄 Live comps (~$0.03)</button></div>
<div class="lcout" id="lc-${p.id}"></div>
</div>`;
}
@@ -217,7 +217,7 @@ function getContext(){
}
const lc = Object.entries(window.liveComps||{}).filter(([,v])=>v&&v.length);
if(lc.length){ lines.push('\nLIVE COMPS PULLED THIS SESSION (fresh web search):');
- lc.forEach(([id,arr])=>{ const p=DATA.ranked.find(x=>x.id===id); lines.push(`for ${p?p.address:id}:`); arr.forEach(c=>lines.push(` - ${c.title} — ${c.snippet}`)); }); }
+ lc.forEach(([id,arr])=>{ const p=DATA.ranked.find(x=>x.id===id); lines.push(`for ${p?p.address:id}:`); arr.forEach(c=>lines.push(` - ${c.title} | ${[c.price,c.cap,c.units,c.sf].filter(Boolean).join(' · ')} | ${c.url}`)); }); }
return lines.join('\n');
}
@@ -279,11 +279,11 @@ async function fetchLiveComps(id){
const r = await fetch('/api/comps',{method:'POST',headers:{'Content-Type':'application/json'},
body:JSON.stringify({address:p.address,city:p.city,type:p.type})});
const j = await r.json();
- if(j.needsKey){ out.innerHTML = `<span class="b warn">⚠ ${j.message}</span>`; return; }
- if(j.error){ out.innerHTML = `<span class="b bad">⚠ ${j.error}</span>`; return; }
+ if(j.error && !(j.comps&&j.comps.length)){ out.innerHTML = `<span class="b warn">⚠ ${j.error}</span>`; return; }
window.liveComps[id] = j.comps||[];
- out.innerHTML = `<div class="small">Live comps (cost ${j.cost}) — ask the chat to compare:</div>` +
- (j.comps||[]).map(c=>`<div class="lc-item"><a href="${c.url}" target="_blank" rel="noopener noreferrer">${c.title||c.url}</a><br>${c.snippet||''}</div>`).join('');
+ const none = (!j.comps||!j.comps.length)?' — none scraped (source may have blocked)':'';
+ out.innerHTML = `<div class="small">Live comps via ${j.source||'web'} (cost ${j.cost})${none} — ask the chat to compare:</div>` +
+ (j.comps||[]).map(c=>`<div class="lc-item"><a href="${c.url}" target="_blank" rel="noopener noreferrer">${c.title||c.url}</a> — ${[c.price,c.cap,c.units,c.sf].filter(Boolean).join(' · ')}</div>`).join('');
}catch(e){ out.innerHTML = `<span class="b bad">⚠ ${e.message}</span>`; }
}
diff --git a/scripts/live-comps-bb.js b/scripts/live-comps-bb.js
new file mode 100644
index 0000000..f3e1b24
--- /dev/null
+++ b/scripts/live-comps-bb.js
@@ -0,0 +1,93 @@
+// live-comps-bb.js — Browserbase-backed live comps. Drives a cloud browser to LoopNet's
+// city+type for-sale search and scrapes the rendered result cards (price / cap / address / url).
+// Usage: CC_CITY="Reseda" CC_TYPE="Multifamily" node scripts/live-comps-bb.js -> prints JSON to stdout.
+const fs = require('fs');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+
+const env = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const KEY = get(env, 'BROWSERBASE_API_KEY'), PROJECT = get(env, 'BROWSERBASE_PROJECT_ID');
+
+const CITY = process.env.CC_CITY || 'Van Nuys';
+const TYPE = (process.env.CC_TYPE || 'Multifamily').toLowerCase();
+// Crexi is more tolerant of cloud browsers than LoopNet (which hard-blocks with Access Denied).
+const SLUG = TYPE.includes('retail') ? 'retail' : TYPE.includes('mixed') ? 'mixed-use' : TYPE.includes('office') ? 'office' : 'multifamily';
+const citySlug = CITY.replace(/\s+/g, '-');
+const URL = `https://www.crexi.com/properties/CA/${citySlug}/${SLUG}`;
+
+(async () => {
+ const out = { source: 'Crexi via Browserbase', url: URL, comps: [] };
+ let browser;
+ try {
+ const bb = new Browserbase({ apiKey: KEY });
+ const session = await bb.sessions.create({ projectId: PROJECT, browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ page.setDefaultTimeout(45000);
+ // Capture the listing JSON the Crexi SPA fetches for itself (authed + geo-scoped) — robust vs DOM scrape.
+ const apiAssets = new Map();
+ page.on('response', async (resp) => {
+ const u = resp.url();
+ if (!u.includes('api.crexi.com')) return;
+ try {
+ const j = await resp.json();
+ const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || []);
+ if (Array.isArray(arr)) for (const a of arr) {
+ const price = a.askingPrice || a.price;
+ const loc = (a.locations && a.locations[0]) || {};
+ if ((a.id || a.assetId) && price && loc.city) {
+ const id = a.id || a.assetId;
+ if (!apiAssets.has(id)) apiAssets.set(id, {
+ title: (a.name || a.description || loc.fullAddress || '').slice(0, 80),
+ url: a.urlSlug ? 'https://www.crexi.com/properties/' + id + '/' + a.urlSlug : 'https://www.crexi.com/properties/' + id,
+ price: '$' + Number(price).toLocaleString(),
+ cap: a.capRate ? a.capRate + '% Cap' : null,
+ units: (String(a.description || '').match(/(\d+)\s*Units?/i) || [])[0] || null,
+ sf: null, city: loc.city, state: (loc.state && loc.state.code) || 'CA'
+ });
+ }
+ }
+ } catch (_) {}
+ });
+ await page.goto(URL, { waitUntil: 'domcontentloaded' });
+ await page.waitForTimeout(7000);
+ for (let i = 0; i < 4; i++) { await page.mouse.wheel(0, 3500).catch(() => {}); await page.waitForTimeout(2200); }
+ const data = await page.evaluate(() => {
+ const body = (document.body.innerText || '');
+ const challenge = /captcha|unusual traffic|verify you are|access denied|are you a robot|blocked/i.test(body) ? (document.title || 'challenge') : null;
+ const anchors = [...document.querySelectorAll('a[href*="/properties/"]')].filter(a => /\/properties\/\d|\/properties\/[a-z]/.test(a.getAttribute('href') || ''));
+ const seen = {}, res = [];
+ for (const a of anchors) {
+ const href = a.href.split('?')[0];
+ if (seen[href]) continue;
+ // climb up to 5 ancestors to find a node whose text carries the price
+ let node = a, txt = '';
+ for (let k = 0; k < 5 && node; k++) { const t = (node.innerText || '').replace(/\s+/g, ' '); if (/\$[\d,]{4,}/.test(t)) { txt = t; break; } node = node.parentElement; }
+ if (!txt) continue;
+ const price = (txt.match(/\$[\d,]{4,}/) || [])[0] || null;
+ if (!price) continue;
+ seen[href] = 1;
+ res.push({
+ title: (a.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 80) || (href.split('/properties/')[1] || href).slice(0, 50),
+ url: href, price,
+ cap: (txt.match(/(\d+\.?\d*)%\s*Cap/i) || [])[0] || null,
+ units: (txt.match(/(\d+)\s*Unit/i) || [])[0] || null,
+ sf: (txt.match(/([\d,]+)\s*SF/i) || [])[0] || null
+ });
+ }
+ return { comps: res.slice(0, 8), challenge, anchorCount: anchors.length, title: document.title };
+ });
+ const apiArr = [...apiAssets.values()];
+ const cityMatch = apiArr.filter(a => a.city && a.city.toLowerCase() === CITY.toLowerCase());
+ const chosen = cityMatch.length ? cityMatch : (data.comps.length ? data.comps : apiArr);
+ out.comps = chosen.slice(0, 8).map(c => ({ title: c.title, url: c.url, price: c.price, cap: c.cap, units: c.units, sf: c.sf }));
+ out.debug = { anchorCount: data.anchorCount, title: data.title, apiAssets: apiArr.length, cityMatch: cityMatch.length };
+ if (cityMatch.length === 0 && apiArr.length) out.note = 'No exact ' + CITY + ' matches in the page feed; showing nearest available.';
+ if (data.challenge && !out.comps.length) out.error = 'Crexi challenge/block hit (' + data.challenge + ')';
+ } catch (e) { out.error = String(e.message || e).split('\n')[0]; }
+ finally { if (browser) await browser.close().catch(() => {}); }
+ process.stdout.write(JSON.stringify(out));
+ process.exit(0);
+})();
diff --git a/scripts/serve.js b/scripts/serve.js
index 116d3b6..bb5d880 100644
--- a/scripts/serve.js
+++ b/scripts/serve.js
@@ -11,28 +11,21 @@ const PORT = process.env.PORT || 9911;
const app = express();
app.use(express.json({ limit: '256kb' }));
-// EXA key for the live-comps route: env var or project .env (gitignored). No key -> route degrades gracefully.
-function exaKey() {
- if (process.env.EXA_API_KEY) return process.env.EXA_API_KEY;
- try { const m = fs.readFileSync(path.join(ROOT, '.env'), 'utf8').match(/^EXA_API_KEY=(.+)$/m); if (m) return m[1].trim(); } catch (_) {}
- return null;
-}
-
-// GATED, cost-surfaced live comps: hits Exa REST (~$0.01/call). Returns {needsKey} if unkeyed.
-app.post('/api/comps', async (req, res) => {
- const { address = '', city = '', type = 'multifamily' } = req.body || {};
- const key = exaKey();
- if (!key) return res.json({ needsKey: true, message: 'Live comps need an Exa API key. Get one at dashboard.exa.ai/api-keys, then add EXA_API_KEY to the project .env (or paste it to Steve to route via the secrets skill).' });
- const query = `recently sold or active comparable ${type} investment property near ${address}, ${city} San Fernando Valley with price, units, and cap rate`;
- try {
- const r = await fetch('https://api.exa.ai/search', {
- method: 'POST', headers: { 'Content-Type': 'application/json', 'x-api-key': key },
- body: JSON.stringify({ query, numResults: 6, type: 'auto', contents: { text: { maxCharacters: 600 } } })
- });
- const j = await r.json();
- const comps = (j.results || []).map(x => ({ title: x.title, url: x.url, snippet: (x.text || '').replace(/\s+/g, ' ').slice(0, 320) }));
- res.json({ comps, cost: '~$0.01', query });
- } catch (e) { res.status(502).json({ error: String(e.message || e) }); }
+// GATED, cost-surfaced live comps via Browserbase (cloud browser -> LoopNet city/type search).
+// ~$0.03/call (a Browserbase session). Shells to scripts/live-comps-bb.js so the cloud-browser
+// deps resolve from the browserbase skill's node_modules.
+const { execFile } = require('child_process');
+app.post('/api/comps', (req, res) => {
+ const { city = 'Van Nuys', type = 'Multifamily' } = req.body || {};
+ const child = execFile('node', [path.join(__dirname, 'live-comps-bb.js')], {
+ env: { ...process.env, CC_CITY: city, CC_TYPE: type, NODE_PATH: process.env.HOME + '/.claude/skills/browserbase/node_modules' },
+ timeout: 90000, maxBuffer: 2 * 1024 * 1024
+ }, (err, stdout) => {
+ if (err && !stdout) return res.status(502).json({ error: 'live comps failed: ' + String(err.message || err).split('\n')[0] });
+ let j; try { j = JSON.parse(stdout.trim().split('\n').pop()); } catch { return res.status(502).json({ error: 'bad comps output' }); }
+ res.json({ comps: j.comps || [], source: j.source, sourceUrl: j.url, error: j.error, cost: '~$0.03 (Browserbase session)' });
+ });
+ child.on('error', e => { try { res.status(502).json({ error: String(e.message) }); } catch (_) {} });
});
// The chat can DRIVE the viewer. This protocol is how it does it.
← 1351a12 Expand listing set (+5) + gated live-comps route
·
back to Commercialrealestate
·
Tighten live-comps units parse 5b97a77 →