← back to Commercialrealestate
TK-10864: LoopNet second-source scraper (broker graph, source=loopnet)
d11b90fad824fa6f96704db19fc09e58bbaf80b6 · 2026-08-26 08:08:41 -0700 · Steve Abrams
Same architecture as the hardened Crexi scraper: openclaw two-level crawl ->
broker_direct_listing, name+firm-confirmed agent attribution via the reliable
'Contact <Agent> of <Firm> by phone' pattern, price from the facts table,
cross-source dedup, resume cache, reversible, never links out. FINDING:
LoopNet gates the listing-agent NAME on ~99% of listings (generic 'Contact
Listing Agent' with no scrapable name), so agent-attribution yield is ~1% vs
Crexi's ~86% — a data-source limitation, not a build defect. Scraper ingests
the exposed-agent subset cleanly.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A scripts/scrape-loopnet.js
Diff
commit d11b90fad824fa6f96704db19fc09e58bbaf80b6
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Aug 26 08:08:41 2026 -0700
TK-10864: LoopNet second-source scraper (broker graph, source=loopnet)
Same architecture as the hardened Crexi scraper: openclaw two-level crawl ->
broker_direct_listing, name+firm-confirmed agent attribution via the reliable
'Contact <Agent> of <Firm> by phone' pattern, price from the facts table,
cross-source dedup, resume cache, reversible, never links out. FINDING:
LoopNet gates the listing-agent NAME on ~99% of listings (generic 'Contact
Listing Agent' with no scrapable name), so agent-attribution yield is ~1% vs
Crexi's ~86% — a data-source limitation, not a build defect. Scraper ingests
the exposed-agent subset cleanly.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/scrape-loopnet.js | 151 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 151 insertions(+)
diff --git a/scripts/scrape-loopnet.js b/scripts/scrape-loopnet.js
new file mode 100644
index 0000000..316904c
--- /dev/null
+++ b/scripts/scrape-loopnet.js
@@ -0,0 +1,151 @@
+#!/usr/bin/env node
+// scrape-loopnet.js — TK-10864. Second source (LoopNet) for the broker-graph master DB, mirroring
+// the hardened Crexi scraper (scrape-crexi-loopnet.js): our-own openclaw two-level crawl → correct
+// per-agent attribution → broker_direct_listing (source='loopnet'), reversible, cross-source deduped,
+// never links out. $0 (openclaw local Chrome). Dry-run default.
+//
+// Recon (2026-08-26): LoopNet is REACHABLE via openclaw (not bot-walled). Search grid is URL-paginated
+// (/for-sale/<page>/). Detail pages expose the listing agent+firm on a subset of listings as the clean
+// line "Contact <Agent> of <Firm> by phone at <phone>"; others show a generic "Contact Listing Agent"
+// with NO scrapable name → we SKIP those (spec: never fabricate an agent). Price from the facts table
+// ("Price\t$X" — NOT the "Price Per Unit"/"Price Per SqFt" distractors). Address carries a ", CA <zip>".
+//
+// Usage: node scripts/scrape-loopnet.js [--type all|Multifamily] [--limit 40] [--pages 8] [--apply] [--reset]
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const db = require('./db/brokers-db');
+const OC = process.env.HOME + '/.npm-global/bin/openclaw';
+const arg = k => (process.argv.find(a => a.startsWith(`--${k}=`))?.split('=')[1]);
+const TYPE_ARG = arg('type') || 'Multifamily';
+const LIMIT = +(arg('limit')) || 40;
+const PAGES = +(arg('pages')) || 8;
+const MINUTES = +(arg('minutes')) || 8;
+const DEADLINE = Date.now() + MINUTES * 60000;
+const APPLY = process.argv.includes('--apply');
+const RESET = process.argv.includes('--reset');
+
+// LoopNet CRE type → URL slug
+const TYPE_SLUG = { 'Multifamily': 'multifamily-properties', 'Retail': 'retail-properties', 'Office': 'office-properties', 'Industrial': 'industrial-properties', 'Mixed Use': 'mixed-use-properties' };
+const ALL_TYPES = Object.keys(TYPE_SLUG);
+const TYPES = TYPE_ARG.toLowerCase() === 'all' ? ALL_TYPES : [TYPE_ARG];
+const searchUrl = (type, page) => `https://www.loopnet.com/search/${TYPE_SLUG[type]}/los-angeles-ca/for-sale/${page > 1 ? page + '/' : ''}`;
+
+const PROG = path.join(__dirname, '..', 'data', 'loopnet-progress.json');
+function loadProg() { if (RESET) return { seenIds: [], seenAddrs: [], runs: [] }; try { const p = JSON.parse(fs.readFileSync(PROG, 'utf8')); p.seenIds ||= []; p.seenAddrs ||= []; p.runs ||= []; return p; } catch { return { seenIds: [], seenAddrs: [], runs: [] }; } }
+function saveProg(p) { try { fs.writeFileSync(PROG, JSON.stringify(p, null, 2)); } catch {} }
+
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+function oc(args, timeout = 55000) { try { return execFileSync(OC, args, { encoding: 'utf8', timeout, stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return null; } }
+function ocEval(fn) { const out = oc(['browser', 'evaluate', '--fn', fn]); if (!out) return null; try { return JSON.parse(out); } catch { return out; } }
+const nAddr = s => String(s || '').toLowerCase().replace(/\b(street|st|avenue|ave|boulevard|blvd|road|rd|drive|dr|lane|ln|place|pl|court|ct|suite|ste|unit|#|north|south|east|west|n|s|e|w)\b/g, '').replace(/[^a-z0-9]+/g, '');
+const nFirm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '').replace(/(inc|llc|corp|corporation|company|co|realty|realestate|group|properties|partners|associates)$/, '');
+const CRE_PRICE_FLOOR = 100000, CRE_PRICE_CEIL = 5000000000;
+
+// ── STEP 1: grid → [{id,url}] (URL-paginated; each page is a fresh navigate) ─────────────────────
+async function extractGrid() {
+ for (let i = 0; i < 3; i++) { oc(['browser', 'evaluate', '--fn', '() => window.scrollTo(0, document.body.scrollHeight)']); await sleep(1200); }
+ const raw = ocEval(`() => { const out=[]; const seen=new Set(); for(const a of document.querySelectorAll('a[href*="/Listing/"]')){ const m=a.href.match(/\\/Listing\\/[^/]+\\/(\\d+)\\//); if(!m) continue; if(seen.has(m[1])) continue; seen.add(m[1]); out.push({id:m[1], url:'https://www.loopnet.com'+a.pathname}); } return out; }`);
+ return Array.isArray(raw) ? raw : [];
+}
+async function collectAllCards(type) {
+ const all = new Map();
+ for (let p = 1; p <= PAGES; p++) {
+ if (!oc(['browser', 'navigate', searchUrl(type, p)])) break;
+ await sleep(3000);
+ const cs = await extractGrid();
+ if (p === 1 && cs.length === 0) return { cards: [], throttled: true };
+ let fresh = 0; for (const c of cs) if (!all.has(c.id)) { all.set(c.id, c); fresh++; }
+ process.stdout.write(` [${type}] page ${p}: +${fresh} (total ${all.size})\n`);
+ if (fresh === 0 && p > 1) break;
+ }
+ return { cards: [...all.values()], throttled: false };
+}
+
+// ── STEP 2: detail → {agent, firm, phone, address, price, cap} ───────────────────────────────────
+function detail(url) {
+ if (!oc(['browser', 'navigate', url])) return null;
+ const d = ocEval(`() => { const t=document.body.innerText||''; return { t: t.slice(0, 9000) }; }`);
+ const t = (d && d.t) || ''; if (!t) return null;
+ const flat = t.replace(/\n+/g, ' ');
+ let address = (flat.match(/([0-9][^,\n]{2,80}?,\s*[A-Za-z .]+,\s*CA\s*\d{5})/) || [])[1] || null;
+ if (address) {
+ // strip a portfolio/marketing prefix like "34 Apartment Buildings California Los Angeles " that
+ // LoopNet prepends before the real "<num> <street>" on multi-building listings.
+ address = address.replace(/^\d+\s+(?:Apartment\s+Buildings?|Buildings?|Units?|Properties)\b.*?(?=\b\d{1,6}\s+[A-Za-z])/i, '').replace(/\s{2,}/g, ' ').trim();
+ }
+ let price = null;
+ const pm = flat.match(/\bPrice\s*\$([\d,]{5,})/) || flat.match(/\$([\d,]{6,})\b/); // facts-table price, then any large $
+ if (pm) { const v = +pm[1].replace(/,/g, ''); if (v >= CRE_PRICE_FLOOR && v <= CRE_PRICE_CEIL) price = v; }
+ // named agent+firm — the reliable "Contact <Agent> of <Firm> by phone" pattern (absent on generic listings → skip)
+ const cm = flat.match(/Contact\s+([A-Z][A-Za-z.'’-]+(?:\s+[A-Z][A-Za-z.'’-]+){1,2})\s+of\s+([A-Za-z0-9.,'’&\- ]{2,60}?)\s+by\s+phone/);
+ const agent = cm ? cm[1].trim() : null;
+ const firm = cm ? cm[2].trim() : null;
+ const cap = (flat.match(/Cap Rate\s+([\d.]+)%/i) || flat.match(/([\d.]+)%\s*Cap/i) || [])[1];
+ return { agent, firm, address, price, cap: cap ? +cap : null };
+}
+
+async function resolveBroker(agent, firm) {
+ if (!agent) return { role: null };
+ const rows = (await db.pool.query(`SELECT b.id, f.name AS firm FROM broker b LEFT JOIN firm f ON f.id=b.firm_id WHERE lower(b.name)=lower($1)`, [agent])).rows;
+ const firmMatch = firm ? rows.filter(r => r.firm && nFirm(r.firm) === nFirm(firm)) : [];
+ if (firmMatch.length === 1) return { brokerId: firmMatch[0].id, role: 'agent' };
+ if (rows.length === 1 && (!firm || !rows[0].firm)) return { brokerId: rows[0].id, role: 'agent' };
+ if (rows.length > 1) return { role: 'firm-listing' };
+ if (!APPLY) return { brokerId: 'NEW', role: 'agent' };
+ let firmId = null;
+ if (firm) { const fr = (await db.pool.query(`INSERT INTO firm(name) VALUES($1) ON CONFLICT(name) DO UPDATE SET name=EXCLUDED.name RETURNING id`, [firm.trim()])).rows[0]; firmId = fr && fr.id; }
+ const ins = (await db.pool.query(`INSERT INTO broker (name, firm_id, source, agent_type, created_at) VALUES ($1,$2,'loopnet','commercial',now()) RETURNING id`, [agent, firmId]).catch(() => ({ rows: [] }))).rows[0];
+ return ins ? { brokerId: ins.id, role: 'agent' } : { role: 'firm-listing' };
+}
+
+(async () => {
+ const prog = loadProg();
+ const seenIds = new Set(prog.seenIds), seenAddrs = new Set(prog.seenAddrs);
+ try { const have = (await db.pool.query(`SELECT address FROM broker_direct_listing`)).rows; for (const r of have) seenAddrs.add(nAddr(r.address)); } catch {}
+ console.log(`\n== LoopNet crawl · [${TYPES.join(', ')}] · LA · ${APPLY ? 'APPLY' : 'DRY-RUN'} · ${PAGES}pg · limit ${LIMIT} ==`);
+ console.log(` resume cache: ${seenIds.size} ids, ${seenAddrs.size} addrs deduped\n`);
+ let wrote = 0, matched = 0, created = 0, firmLevel = 0, crawled = 0, noAgent = 0, consecNull = 0;
+ const checkpoint = () => { if (APPLY) { prog.seenIds = [...seenIds]; prog.seenAddrs = [...seenAddrs]; saveProg(prog); } };
+ for (const type of TYPES) {
+ if (crawled >= LIMIT || Date.now() > DEADLINE) break;
+ const { cards, throttled } = await collectAllCards(type);
+ if (throttled) { console.log(` ⚠ [${type}] throttled/empty — skip`); continue; }
+ console.log(` [${type}] grid: ${cards.length} cards\n`);
+ for (const c of cards) {
+ if (crawled >= LIMIT) break;
+ if (Date.now() > DEADLINE) { console.log(` ⏱ ${MINUTES}m budget reached — stopping cleanly.`); break; }
+ if (seenIds.has(c.id)) continue;
+ const det = detail(c.url);
+ if (!det) { if (++consecNull >= 5) { console.log(' ⚠ 5 blank pages — cooling down.'); break; } continue; }
+ consecNull = 0; crawled++; seenIds.add(c.id); await sleep(1500);
+ const address = det.address, price = det.price, agent = det.agent, firm = det.firm;
+ if (!address) { console.log(` · [skip] ${c.id} — no CA address`); continue; }
+ if (seenAddrs.has(nAddr(address))) { continue; } // cross-source dedup
+ if (!price) { console.log(` · [skip] ${address} — no price`); continue; }
+ if (!agent) { noAgent++; console.log(` · [skip] ${address} — no named listing agent`); continue; }
+ const { brokerId, role } = await resolveBroker(agent, firm);
+ if (!role || (role === 'firm-listing' && !firm)) { console.log(` · [skip] ${address} — ${agent} unresolvable`); continue; }
+ const tag = role === 'firm-listing' ? '[firm-level]' : brokerId === 'NEW' ? '[would create]' : brokerId ? '' : '[no broker]';
+ console.log(` ✓ $${price.toLocaleString()} ${address} — ${agent} @ ${firm || '?'} ${tag}`);
+ if (role === 'firm-listing') firmLevel++; else if (brokerId === 'NEW') created++; else if (brokerId) matched++;
+ if (APPLY && (role === 'firm-listing' ? firm : (brokerId && brokerId !== 'NEW'))) {
+ const bid = role === 'firm-listing' ? null : brokerId;
+ const ins = (await db.pool.query(
+ `INSERT INTO broker_direct_listing (broker_id,agent_name,firm_name,address,price,type,cap_rate,role,source,detail_url)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,$8,'loopnet',$9) ON CONFLICT DO NOTHING RETURNING id`,
+ [bid, agent, firm, address, price, type, det.cap, role, c.url]).catch(() => ({ rows: [] }))).rows[0];
+ if (ins) { wrote++; seenAddrs.add(nAddr(address)); if (wrote % 10 === 0) checkpoint(); }
+ }
+ }
+ }
+ prog.seenIds = [...seenIds]; prog.seenAddrs = [...seenAddrs];
+ prog.runs.push({ ts: new Date().toISOString(), types: TYPES, apply: APPLY, crawled, wrote, matched, created, firmLevel });
+ if (prog.runs.length > 50) prog.runs = prog.runs.slice(-50);
+ if (APPLY || RESET) saveProg(prog);
+ const total = APPLY ? (await db.pool.query(`SELECT count(*) FROM broker_direct_listing WHERE source='loopnet'`)).rows[0].count : 'dry';
+ console.log(`\n== ${crawled} details · ${matched} matched · ${created} ${APPLY ? 'created' : 'would create'} · ${firmLevel} firm-level · ${noAgent} no-named-agent · ${APPLY ? wrote + ' written' : 'DRY-RUN'} ==`);
+ console.log(` loopnet total in DB now: ${total} · Cost: $0 (openclaw local)`);
+ console.log(APPLY ? " Undo: DELETE FROM broker_direct_listing WHERE source='loopnet';" : ' Re-run with --apply to write.');
+ process.exit(0);
+})();
← 777bca5 auto-data-snapshot: 2026-08-26T07:08:04 (1 data files) — dat
·
back to Commercialrealestate
·
chore: lint + quality-gate fixes, v0.25.0 (session close) 9198a6e →