← back to Commercialrealestate
TK-10709: robust Crexi agent-attributed scraper
f0860c198bacc19264afdd8562dd2a3437624da5 · 2026-08-25 14:56:35 -0700 · Steve Abrams
Reliable detail-page extraction (clean address, real headline price not
per-unit/psf, correct primary agent+firm from the structured Listing
Contacts block). Fixes found live: broken new-broker insert (integer seq
not uuid), per-unit price misparse, firm=license-string, /LIC/ regex
eating firms like Mil-lic-hap. Robustness: resume cache w/ incremental
checkpoint, throttle detection + cool-down, soft time budget for clean
exit, persisted grid queue (skip re-sweep), cheap non-CA grid pre-filter,
firm-level fallback. Reversible (source=crexi), never links out.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M scripts/scrape-crexi-loopnet.js
Diff
commit f0860c198bacc19264afdd8562dd2a3437624da5
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Tue Aug 25 14:56:35 2026 -0700
TK-10709: robust Crexi agent-attributed scraper
Reliable detail-page extraction (clean address, real headline price not
per-unit/psf, correct primary agent+firm from the structured Listing
Contacts block). Fixes found live: broken new-broker insert (integer seq
not uuid), per-unit price misparse, firm=license-string, /LIC/ regex
eating firms like Mil-lic-hap. Robustness: resume cache w/ incremental
checkpoint, throttle detection + cool-down, soft time budget for clean
exit, persisted grid queue (skip re-sweep), cheap non-CA grid pre-filter,
firm-level fallback. Reversible (source=crexi), never links out.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/scrape-crexi-loopnet.js | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/scripts/scrape-crexi-loopnet.js b/scripts/scrape-crexi-loopnet.js
index c710886..150ef0b 100644
--- a/scripts/scrape-crexi-loopnet.js
+++ b/scripts/scrape-crexi-loopnet.js
@@ -77,7 +77,7 @@ const cleanAddr = a => a && a.replace(/^0+\s+/, '').replace(/^[^0-9]*(\d)/, '$1'
// ── STEP 1: search grid -> [{url,id,price,address,cap}]. Scroll the SPA to load lazy cards. ───────
async function extractCards() {
- for (let i = 0; i < 3; i++) { oc(['browser', 'evaluate', '--fn', '() => window.scrollTo(0, document.body.scrollHeight)']); await sleep(1200); }
+ for (let i = 0; i < 4; i++) { oc(['browser', 'evaluate', '--fn', '() => window.scrollTo(0, document.body.scrollHeight)']); await sleep(1200); }
const raw = ocEval(`() => {
const out=[]; const seen=new Set();
for (const a of document.querySelectorAll('a[href*="/properties/"]')) {
@@ -85,7 +85,7 @@ async function extractCards() {
let card=a, el=a;
while(el.parentElement && el.parentElement.querySelectorAll('a[href*="/properties/"]').length<=1){ el=el.parentElement; card=el; }
seen.add(id);
- out.push({ id, text: (card.textContent||'').replace(/\\s+/g,' ').trim().slice(0,300) });
+ out.push({ id, text: (card.textContent||'').replace(/\\s+/g,' ').trim().slice(0,400) });
}
return out;
}`);
@@ -94,7 +94,8 @@ async function extractCards() {
const price = parsePrice(r.text);
const addr = cleanAddr((r.text.match(ADDR_RE) || [])[0]);
const cap = (r.text.match(/([0-9.]+)%\s*CAP/i) || [])[1];
- cards.push({ url: 'https://www.crexi.com/properties/' + r.id + '/', id: r.id, price, address: addr || null, cap: cap ? +cap : null });
+ const state = (r.text.match(/,\s*([A-Z]{2})\s*\d{5}/) || [])[1] || null; // grid-level state for cheap non-CA pre-filter
+ cards.push({ url: 'https://www.crexi.com/properties/' + r.id + '/', id: r.id, price, address: addr || null, state, cap: cap ? +cap : null });
}
return cards;
}
@@ -208,7 +209,7 @@ async function resolveBroker(agent, firm, license, phone) {
console.log(`\n== Crexi crawl · [${TYPES.join(', ')}] · LA · ${APPLY ? 'APPLY' : 'DRY-RUN'} · ${PAGES}pg · limit ${LIMIT} new details ==`);
console.log(` resume cache: ${seenIds.size} ids seen, ${seenAddrs.size} addrs ingested (target ≥500)\n`);
- let wrote = 0, matched = 0, created = 0, firmLevel = 0, crawled = 0, unpriced = 0, throttledTypes = 0, consecNull = 0;
+ let wrote = 0, matched = 0, created = 0, firmLevel = 0, crawled = 0, unpriced = 0, throttledTypes = 0, consecNull = 0, nonCA = 0;
const checkpoint = () => { if (APPLY) { prog.seenIds = [...seenIds]; prog.seenAddrs = [...seenAddrs]; saveProg(prog); } };
let deadlineHit = false;
for (const type of TYPES) {
@@ -224,6 +225,9 @@ async function resolveBroker(agent, firm, license, phone) {
if (crawled >= LIMIT) break;
if (Date.now() > DEADLINE) { deadlineHit = true; console.log(` ⏱ ${MINUTES}m soft budget reached — stopping cleanly (progress saved).`); break; }
if (seenIds.has(c.id)) continue; // resume: already crawled in a prior run
+ // CHEAP non-CA pre-filter: the LA-placeId grid mixes in out-of-area listings (NY/NC/etc.).
+ // If the grid card names a non-CA state, skip WITHOUT the ~15s detail crawl (mark seen once).
+ if (c.state && c.state !== 'CA') { seenIds.add(c.id); nonCA++; continue; }
// early skip if the grid address is one we already ingested (avoids a wasted detail hit)
if (c.address && seenAddrs.has(nAddr(c.address))) { seenIds.add(c.id); continue; }
const det = detailAgent(c.url);
@@ -270,7 +274,7 @@ async function resolveBroker(agent, firm, license, phone) {
if (APPLY || RESET) saveProg(prog);
const total = APPLY ? (await db.pool.query(`SELECT count(*) FROM broker_direct_listing WHERE source='crexi'`)).rows[0].count : 'dry';
- console.log(`\n== ${crawled} new details crawled · ${matched} matched · ${created} ${APPLY ? 'brokers created' : 'would create'} · ${firmLevel} firm-level · ${APPLY ? wrote + ' written' : 'DRY-RUN'} ==`);
+ console.log(`\n== ${crawled} new details crawled · ${matched} matched · ${created} ${APPLY ? 'brokers created' : 'would create'} · ${firmLevel} firm-level · ${nonCA} non-CA pre-filtered · ${APPLY ? wrote + ' written' : 'DRY-RUN'} ==`);
console.log(` crexi total in DB now: ${total} / 500 target · resume cache saved (${seenIds.size} ids) · Cost: $0 (openclaw local)`);
if (throttledTypes) console.log(` ⚠ ${throttledTypes} type(s) throttled — re-run after a cool-down; already-crawled pages will be skipped.`);
console.log(APPLY ? " Undo: DELETE FROM broker_direct_listing WHERE source='crexi' AND scraped_at::date=CURRENT_DATE;" : ' Re-run with --apply to write.');
← 7361526 afternoon CRE update 2026-08-25
·
back to Commercialrealestate
·
TK-10709: strip Crexi marketing-header prefix from detail ad 963bc1d →