[object Object]

← back to Commercialrealestate

chore: lint + quality-gate fixes, v0.25.0 (session close)

9198a6eb1af3c9a6adebab622c7fd7aef0609df2 · 2026-08-26 08:13:53 -0700 · Steve Abrams

crexi: drop dead --type bare-flag branch (TYPE_ARG precedence). loopnet:
tighten price to facts-table headline only (no loose $-fallback that could
misparse per-unit/psf), add deadlineHit outer-break to match crexi. Behavior-
preserving; both parse clean. [lint ✓ · refactor ✓ (clean) · v0.24.1→v0.25.0]

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

Diff

commit 9198a6eb1af3c9a6adebab622c7fd7aef0609df2
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 26 08:13:53 2026 -0700

    chore: lint + quality-gate fixes, v0.25.0 (session close)
    
    crexi: drop dead --type bare-flag branch (TYPE_ARG precedence). loopnet:
    tighten price to facts-table headline only (no loose $-fallback that could
    misparse per-unit/psf), add deadlineHit outer-break to match crexi. Behavior-
    preserving; both parse clean. [lint ✓ · refactor ✓ (clean) · v0.24.1→v0.25.0]
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 package.json                    | 2 +-
 scripts/scrape-crexi-loopnet.js | 2 +-
 scripts/scrape-loopnet.js       | 9 ++++++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/package.json b/package.json
index 6434822..3960204 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "commercialrealestate",
-  "version": "0.24.1",
+  "version": "0.25.0",
   "private": true,
   "description": "LA County CRE investment explorer — multi-firm sourced, Census + assessor enriched, Qwen-analyzed",
   "scripts": {
diff --git a/scripts/scrape-crexi-loopnet.js b/scripts/scrape-crexi-loopnet.js
index 02e2ef6..bb0f1d2 100644
--- a/scripts/scrape-crexi-loopnet.js
+++ b/scripts/scrape-crexi-loopnet.js
@@ -27,7 +27,7 @@ const { execFileSync } = require('child_process');
 const db = require('./db/brokers-db');
 const OC = process.env.HOME + '/.npm-global/bin/openclaw';
 const arg = k => (process.argv.find(a => a.startsWith(`--${k}=`))?.split('=')[1]);
-const TYPE_ARG = arg('type') || process.argv.includes('--type') ? (arg('type') || 'Multifamily') : 'Multifamily';
+const TYPE_ARG = arg('type') || 'Multifamily';
 const LIMIT = +(arg('limit')) || 40;          // NEW (unseen) detail pages to crawl this run (polite)
 const PAGES = +(arg('pages')) || 30;          // search pages to sweep per type
 const APPLY = process.argv.includes('--apply');
diff --git a/scripts/scrape-loopnet.js b/scripts/scrape-loopnet.js
index 316904c..9078677 100644
--- a/scripts/scrape-loopnet.js
+++ b/scripts/scrape-loopnet.js
@@ -75,7 +75,9 @@ function detail(url) {
     address = address.replace(/^\d+\s+(?:Apartment\s+Buildings?|Buildings?|Units?|Properties)\b.*?(?=\b\d{1,6}\s+[A-Za-z])/i, '').replace(/\s{2,}/g, ' ').trim();
   }
   let price = null;
-  const pm = flat.match(/\bPrice\s*\$([\d,]{5,})/) || flat.match(/\$([\d,]{6,})\b/); // facts-table price, then any large $
+  // facts-table headline price ONLY. No loose "$[6+ digits]" fallback — that would grab a
+  // per-unit / price-per-sqft number and misparse; better to skip a listing with no clean Price.
+  const pm = flat.match(/\bPrice\s*\$([\d,]{5,})/);
   if (pm) { const v = +pm[1].replace(/,/g, ''); if (v >= CRE_PRICE_FLOOR && v <= CRE_PRICE_CEIL) price = v; }
   // named agent+firm — the reliable "Contact <Agent> of <Firm> by phone" pattern (absent on generic listings → skip)
   const cm = flat.match(/Contact\s+([A-Z][A-Za-z.'’-]+(?:\s+[A-Z][A-Za-z.'’-]+){1,2})\s+of\s+([A-Za-z0-9.,'’&\- ]{2,60}?)\s+by\s+phone/);
@@ -105,7 +107,7 @@ async function resolveBroker(agent, firm) {
   try { const have = (await db.pool.query(`SELECT address FROM broker_direct_listing`)).rows; for (const r of have) seenAddrs.add(nAddr(r.address)); } catch {}
   console.log(`\n== LoopNet crawl · [${TYPES.join(', ')}] · LA · ${APPLY ? 'APPLY' : 'DRY-RUN'} · ${PAGES}pg · limit ${LIMIT} ==`);
   console.log(`   resume cache: ${seenIds.size} ids, ${seenAddrs.size} addrs deduped\n`);
-  let wrote = 0, matched = 0, created = 0, firmLevel = 0, crawled = 0, noAgent = 0, consecNull = 0;
+  let wrote = 0, matched = 0, created = 0, firmLevel = 0, crawled = 0, noAgent = 0, consecNull = 0, deadlineHit = false;
   const checkpoint = () => { if (APPLY) { prog.seenIds = [...seenIds]; prog.seenAddrs = [...seenAddrs]; saveProg(prog); } };
   for (const type of TYPES) {
     if (crawled >= LIMIT || Date.now() > DEADLINE) break;
@@ -114,7 +116,7 @@ async function resolveBroker(agent, firm) {
     console.log(`  [${type}] grid: ${cards.length} cards\n`);
     for (const c of cards) {
       if (crawled >= LIMIT) break;
-      if (Date.now() > DEADLINE) { console.log(`  ⏱ ${MINUTES}m budget reached — stopping cleanly.`); break; }
+      if (Date.now() > DEADLINE) { deadlineHit = true; console.log(`  ⏱ ${MINUTES}m budget reached — stopping cleanly.`); break; }
       if (seenIds.has(c.id)) continue;
       const det = detail(c.url);
       if (!det) { if (++consecNull >= 5) { console.log('  ⚠ 5 blank pages — cooling down.'); break; } continue; }
@@ -138,6 +140,7 @@ async function resolveBroker(agent, firm) {
         if (ins) { wrote++; seenAddrs.add(nAddr(address)); if (wrote % 10 === 0) checkpoint(); }
       }
     }
+    if (deadlineHit) break;
   }
   prog.seenIds = [...seenIds]; prog.seenAddrs = [...seenAddrs];
   prog.runs.push({ ts: new Date().toISOString(), types: TYPES, apply: APPLY, crawled, wrote, matched, created, firmLevel });

← d11b90f TK-10864: LoopNet second-source scraper (broker graph, sourc  ·  back to Commercialrealestate  ·  SFV $700k-$1.2M pool split: enrichment + /api/sfv-pool + det 548c019 →