← back to Commercialrealestate
CRCP multi-metro listings sweep: generalize SFV Crexi scrape to all 21 metros + per-metro config; paid run gated
367513f18c2561e5dd0508bbcd9d4eca3c131eb6 · 2026-07-12 11:01:10 -0700 · Steve
Files touched
A data/metro-listings-config.jsonA scripts/refresh-listings-multi.js
Diff
commit 367513f18c2561e5dd0508bbcd9d4eca3c131eb6
Author: Steve <steve@designerwallcoverings.com>
Date: Sun Jul 12 11:01:10 2026 -0700
CRCP multi-metro listings sweep: generalize SFV Crexi scrape to all 21 metros + per-metro config; paid run gated
---
data/metro-listings-config.json | 28 ++++++++++
scripts/refresh-listings-multi.js | 112 ++++++++++++++++++++++++++++++++++++++
2 files changed, 140 insertions(+)
diff --git a/data/metro-listings-config.json b/data/metro-listings-config.json
new file mode 100644
index 0000000..de8a2f4
--- /dev/null
+++ b/data/metro-listings-config.json
@@ -0,0 +1,28 @@
+{
+ "_note": "Per-metro Crexi sweep config for refresh-listings-multi.js. Each metro: the Crexi state code, a list of city pages to navigate, and a commercial price band (min/max) appropriate to that market. Price bands are wide on purpose — the goal is market-activity coverage for the agent-targeting product, not a personal buy-box. Adjust freely.",
+ "types": [["multifamily", "Multifamily"], ["retail", "Retail"], ["office", "Office"]],
+ "metros": {
+ "los-angeles": { "state": "CA", "min": 750000, "max": 40000000, "cities": ["Los Angeles", "Long Beach", "Pasadena", "Santa Monica", "Culver City", "Glendale", "Inglewood", "Torrance"] },
+ "beverly-hills": { "state": "CA", "min": 1500000, "max": 100000000, "cities": ["Beverly Hills", "West Hollywood", "Bel Air"] },
+ "malibu": { "state": "CA", "min": 1500000, "max": 100000000, "cities": ["Malibu"] },
+ "newport-beach": { "state": "CA", "min": 1000000, "max": 80000000, "cities": ["Newport Beach", "Corona Del Mar", "Costa Mesa"] },
+ "montecito": { "state": "CA", "min": 1000000, "max": 80000000, "cities": ["Santa Barbara", "Montecito"] },
+ "san-francisco": { "state": "CA", "min": 1000000, "max": 100000000, "cities": ["San Francisco"] },
+ "silicon-valley": { "state": "CA", "min": 1000000, "max": 100000000, "cities": ["Palo Alto", "Menlo Park", "Mountain View", "San Jose", "Sunnyvale", "Cupertino"] },
+ "atherton": { "state": "CA", "min": 1500000, "max": 100000000, "cities": ["Atherton", "Redwood City"] },
+ "hillsborough": { "state": "CA", "min": 1500000, "max": 100000000, "cities": ["Hillsborough", "San Mateo", "Burlingame"] },
+ "woodside": { "state": "CA", "min": 1500000, "max": 100000000, "cities": ["Woodside", "Portola Valley"] },
+ "miami": { "state": "FL", "min": 750000, "max": 100000000, "cities": ["Miami", "Coral Gables", "Coconut Grove"] },
+ "miami-beach": { "state": "FL", "min": 1000000, "max": 100000000, "cities": ["Miami Beach"] },
+ "palm-beach": { "state": "FL", "min": 1000000, "max": 100000000, "cities": ["Palm Beach", "West Palm Beach"] },
+ "naples": { "state": "FL", "min": 750000, "max": 80000000, "cities": ["Naples"] },
+ "nyc": { "state": "NY", "min": 1500000, "max": 250000000, "cities": ["New York", "Brooklyn"] },
+ "greenwich": { "state": "CT", "min": 1000000, "max": 80000000, "cities": ["Greenwich"] },
+ "hamptons": { "state": "NY", "min": 1000000, "max": 100000000, "cities": ["East Hampton", "Southampton", "Bridgehampton"] },
+ "las-vegas": { "state": "NV", "min": 750000, "max": 100000000, "cities": ["Las Vegas", "Henderson"] },
+ "texas": { "state": "TX", "min": 750000, "max": 100000000, "cities": ["Austin", "Dallas", "Houston", "San Antonio", "Fort Worth"] },
+ "aspen": { "state": "CO", "min": 1500000, "max": 100000000, "cities": ["Aspen", "Snowmass Village"] },
+ "vail": { "state": "CO", "min": 1000000, "max": 80000000, "cities": ["Vail", "Beaver Creek"] },
+ "telluride": { "state": "CO", "min": 1000000, "max": 80000000, "cities": ["Telluride"] }
+ }
+}
diff --git a/scripts/refresh-listings-multi.js b/scripts/refresh-listings-multi.js
new file mode 100644
index 0000000..82ea98a
--- /dev/null
+++ b/scripts/refresh-listings-multi.js
@@ -0,0 +1,112 @@
+// refresh-listings-multi.js — generalize the SFV Crexi sweep to ALL 21 CRCP metros.
+// Reads data/metro-listings-config.json (per-metro cities + price band), sweeps each
+// metro's city/type pages in ONE Browserbase session, captures Crexi's api.crexi.com
+// asset JSON, dedupes by address, TAGS each listing with its `metro` (feeds the
+// email-alerts metro filter), and APPENDS to data/listings.json.
+//
+// ⚠ PAID: uses Browserbase (metered cloud browser, solveCaptchas on). Gated.
+// Estimate cost before running: ~1 session, ~(sum of cities × types) page navs.
+//
+// Usage:
+// node scripts/refresh-listings-multi.js # ALL metros (paid, gated)
+// node scripts/refresh-listings-multi.js los-angeles # one metro
+// CC_LOCAL=1 node scripts/refresh-listings-multi.js aspen # try local Chrome ($0, may hit anti-bot)
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+
+const ROOT = path.join(__dirname, '..');
+const CFG = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'metro-listings-config.json'), 'utf8'));
+const ONLY = process.argv[2]; // optional single metro slug
+const LOCAL = process.env.CC_LOCAL === '1';
+const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9 ]/g, '').replace(/\b(ave|avenue|st|street|blvd|boulevard|dr|drive|rd|road|ln|lane|pl|place|ct|court)\b/g, '').replace(/\s+/g, ' ').trim();
+
+async function makeBrowser() {
+ if (LOCAL) {
+ // $0 local Chrome. Crexi is anti-bot-walled, so this may capture little — probe only.
+ const execPath = process.env.CHROME_PATH || '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+ const browser = await chromium.launch({ headless: true, executablePath: execPath });
+ return { browser, page: await (await browser.newContext({ viewport: { width: 1440, height: 1000 } })).newPage(), local: true };
+ }
+ const Browserbase = require('@browserbasehq/sdk').default;
+ const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+ const get = (t, k) => (t.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+ const bb = new Browserbase({ apiKey: get(bbEnv, 'BROWSERBASE_API_KEY') });
+ const session = await bb.sessions.create({ projectId: get(bbEnv, 'BROWSERBASE_PROJECT_ID'), browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ console.log('bb session', session.id);
+ const browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ return { browser, page: ctx.pages()[0] || await ctx.newPage(), local: false };
+}
+
+(async () => {
+ const metros = ONLY ? { [ONLY]: CFG.metros[ONLY] } : CFG.metros;
+ if (ONLY && !CFG.metros[ONLY]) { console.error(`unknown metro: ${ONLY}`); process.exit(1); }
+ const assets = new Map(); // id -> {..., metro}
+ let browser;
+ try {
+ const b = await makeBrowser(); browser = b.browser; const page = b.page;
+ page.setDefaultTimeout(45000);
+ page.on('response', async (resp) => {
+ if (!resp.url().includes('api.crexi.com')) return;
+ try {
+ const j = await resp.json();
+ const arr = Array.isArray(j) ? j : (j.data || j.assets || j.results || []);
+ if (Array.isArray(arr)) for (const a of arr) {
+ const id = a.id || a.assetId, price = a.askingPrice || a.price, loc = (a.locations && a.locations[0]) || {};
+ if (id && price && loc.city && !assets.has(id)) assets.set(id, {
+ id, price: +price, city: loc.city, address: loc.address || (a.name || ''),
+ cap: typeof a.capRate === 'number' ? a.capRate : null, urlSlug: a.urlSlug || '',
+ desc: String(a.description || ''), types: a.types || [], metro: null,
+ });
+ }
+ } catch (_) {}
+ });
+
+ for (const [slug, cfg] of Object.entries(metros)) {
+ const cityset = new Set(cfg.cities.map(c => c.toLowerCase()));
+ const before = assets.size;
+ for (const [tslug] of CFG.types) {
+ for (const city of cfg.cities) {
+ const url = `https://www.crexi.com/properties/${cfg.state}/${city.replace(/\s+/g, '-')}/${tslug}`;
+ try { await page.goto(url, { waitUntil: 'domcontentloaded' }); await page.waitForTimeout(4500); await page.mouse.wheel(0, 3000).catch(() => {}); await page.waitForTimeout(1500); }
+ catch (e) { console.log('nav err', slug, city, tslug, e.message.split('\n')[0]); }
+ }
+ }
+ // tag any newly-captured, in-city, in-band asset with this metro
+ for (const a of assets.values()) {
+ if (a.metro) continue;
+ if (cityset.has((a.city || '').toLowerCase()) && a.price >= cfg.min && a.price <= cfg.max) a.metro = slug;
+ }
+ console.log(` ${slug}: session now holds ${assets.size} assets (+${assets.size - before} nav'd)`);
+ }
+ } catch (e) { console.error('FATAL', e.message); }
+ finally { if (browser) await browser.close().catch(() => {}); }
+
+ const cand = [...assets.values()].filter(a => a.metro && a.address);
+ console.log(`captured ${assets.size} assets, ${cand.length} metro-tagged candidates`);
+
+ const data = JSON.parse(fs.readFileSync(path.join(ROOT, 'data', 'listings.json'), 'utf8'));
+ const have = new Set(data.listings.map(l => norm(l.address)));
+ const today = new Date().toISOString().slice(0, 10);
+ let added = 0;
+ for (const a of cand) {
+ const key = norm(a.address);
+ if (!key || have.has(key)) continue;
+ have.add(key);
+ const type = (a.types || []).some(t => /retail/i.test(t)) ? 'Retail' : (a.types || []).some(t => /office/i.test(t)) ? 'Office' : (a.types || []).some(t => /mixed/i.test(t)) ? 'Mixed-use' : 'Multifamily';
+ const units = (() => { const m = a.desc.match(/(?:\|\s*)?\b(\d{1,3})\s*units?\b/i); if (m) { const n = +m[1]; if (n >= 1 && n <= 500 && a.price / n >= 50000) return n; } return null; })();
+ data.listings.push({
+ id: 'crx' + a.id, address: a.address, city: a.city, metro: a.metro, zip: '', type,
+ price: a.price, units: units || 1, sqft: null, cap_rate: a.cap, verified: false,
+ year_built: null, rent_control: 'verify', status: 'Active',
+ upside_note: `Auto-added by multi-metro Crexi sweep ${today} (${a.metro}). ${a.cap ? 'Broker cap ' + a.cap + '% (unverified).' : 'Cap not disclosed.'} Verify before acting.`,
+ source: 'https://www.crexi.com/properties/' + a.id + (a.urlSlug ? '/' + a.urlSlug : ''),
+ });
+ added++;
+ }
+ data.meta = data.meta || {};
+ data.meta.multi_metro_last_run = { date: today, captured: assets.size, candidates: cand.length, added, metros: Object.keys(metros) };
+ fs.writeFileSync(path.join(ROOT, 'data', 'listings.json'), JSON.stringify(data, null, 2));
+ console.log(`ADDED ${added} new listings (total ${data.listings.length}).`);
+})();
← 91621f0 CRCP email-alerts: snapshot-diff engine + George send wiring
·
back to Commercialrealestate
·
Add spreadsheet-style draggable column resizing (col-resize. dd4abf6 →