CRCP crexi DB scraper: parse openclaw CLI output past its stdout preamble, 'Listed by' as primary firm source, measured 0-card diagnostics, 6s SPA settle (TK-12033)

5d3636732255f51e1329e0748ad1cf2ecae07ea4 · 2026-09-23 12:08:23 -0700 · Steve Abrams

The openclaw CLI printed a Config-warnings box on STDOUT before the JSON result, so every
evaluate() parsed as a string -> 0 cards -> a false THROTTLED verdict (the same 'inferred from an
empty parse' class as the extractor bug). The 2026-09 detail page adds masked/PRO lines that the
reverse line-scan misread as the firm; the structured 'Listed by <firm>' line is now primary.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01X7HcZ26MePoJ5QpcvDhKmH

Files touched

M scripts/scrape-crexi-loopnet.js

Diff

commit 5d3636732255f51e1329e0748ad1cf2ecae07ea4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Sep 23 12:08:23 2026 -0700

    CRCP crexi DB scraper: parse openclaw CLI output past its stdout preamble, 'Listed by' as primary firm source, measured 0-card diagnostics, 6s SPA settle (TK-12033)
    
    The openclaw CLI printed a Config-warnings box on STDOUT before the JSON result, so every
    evaluate() parsed as a string -> 0 cards -> a false THROTTLED verdict (the same 'inferred from an
    empty parse' class as the extractor bug). The 2026-09 detail page adds masked/PRO lines that the
    reverse line-scan misread as the firm; the structured 'Listed by <firm>' line is now primary.
    
    Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
    Claude-Session: https://claude.ai/code/session_01X7HcZ26MePoJ5QpcvDhKmH
---
 scripts/scrape-crexi-loopnet.js | 38 +++++++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/scripts/scrape-crexi-loopnet.js b/scripts/scrape-crexi-loopnet.js
index bcc74b6..e4f46e7 100644
--- a/scripts/scrape-crexi-loopnet.js
+++ b/scripts/scrape-crexi-loopnet.js
@@ -57,7 +57,19 @@ function saveProg(p) { try { fs.writeFileSync(PROG, JSON.stringify(p, null, 2));
 
 const sleep = ms => new Promise(r => setTimeout(r, ms));
 function oc(args, timeout = 45000) { try { return execFileSync(OC, args, { encoding: 'utf8', timeout, stdio: ['ignore', 'pipe', 'ignore'] }); } catch { return null; } }
-function ocEval(fn) { const out = oc(['browser', 'evaluate', '--fn', fn]); if (!out) return null; try { return JSON.parse(out); } catch { return out; } }
+// The openclaw CLI (2026.3.x) prints a "Config warnings" preamble to STDOUT before the JSON result, which made a
+// bare JSON.parse fail -> every card sweep read as [] -> a false "THROTTLED" (TK-12033). Parse from the first JSON line.
+function parseOcJson(out) {
+  if (!out) return null;
+  try { return JSON.parse(out); } catch {}
+  const lines = out.split("\n");
+  for (let i = 0; i < lines.length; i++) {
+    const s = lines.slice(i).join("\n").trim();
+    if (/^[\[{"]/.test(s) || /^(true|false|null|-?\d)/.test(s)) { try { return JSON.parse(s); } catch {} }
+  }
+  return out;
+}
+function ocEval(fn) { const out = oc(['browser', 'evaluate', '--fn', fn]); if (!out) return null; return parseOcJson(out); }
 const nAddr = s => String(s || '').toLowerCase().replace(/\b(street|st|avenue|ave|boulevard|blvd|road|rd|drive|dr|lane|ln|place|pl|court|ct|suite|ste|unit|#|north|south|east|west|n|s|e|w)\b/g, '').replace(/[^a-z0-9]+/g, '');
 const nFirm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]+/g, '').replace(/(inc|llc|corp|corporation|company|co|realty|realestate|group|properties|partners|associates)$/, '');
 
@@ -114,12 +126,18 @@ async function collectAllCards(type) {
     return { cards: cached.byType[type], throttled: false };
   }
   const all = new Map();
-  if (!oc(['browser', 'navigate', searchUrl(type)])) return { cards: [], throttled: true };
-  await sleep(2500);
+  if (!oc(['browser', 'navigate', searchUrl(type)])) { console.log(`  [${type}] navigate FAILED (openclaw CLI returned nothing) — ${searchUrl(type)}`); return { cards: [], throttled: true }; }
+  await sleep(6000);                                     // SPA settle: the v2 grid needs ~5-6s before cards render (measured 2026-09-23)
   for (let p = 1; p <= PAGES; p++) {
     const cs = await extractCards();
     // THROTTLE signature: page 1 yields nothing rendered at all.
-    if (p === 1 && cs.length === 0) return { cards: [], throttled: true };
+    if (p === 1 && cs.length === 0) {
+      // MEASURE, don't infer (TK-11431 class): a nav failure, a slow SPA, a bot wall and a real throttle all
+      // look like "0 cards". Print what the tab actually holds so the operator can tell them apart.
+      const diag = ocEval(`() => ({ url: location.href, title: document.title.slice(0,80), anchors: document.querySelectorAll('a[href]').length, propLinks: document.querySelectorAll('a[href*="/properties/"]').length, hint: (document.body.innerText||'').replace(/\\s+/g,' ').slice(0,160) })`);
+      console.log(`  [${type}] 0 cards — page state: ${typeof diag === 'object' ? JSON.stringify(diag) : String(diag).slice(0,200)}`);
+      return { cards: [], throttled: true };
+    }
     let fresh = 0; for (const c of cs) if (!all.has(c.id)) { all.set(c.id, c); fresh++; }
     process.stdout.write(`  [${type}] page ${p}: +${fresh} (total ${all.size})\n`);
     if (fresh === 0 && p > 1) break;
@@ -162,11 +180,17 @@ function detailAgent(url) {
   let agent = null, firm = null, license = null;
   const chunk = ((d.lc || head).split(/View Profile/)[0]) || '';
   const lines = chunk.split('\n').map(s => s.trim()).filter(Boolean)
-    .filter(l => !/^(Listing Contacts|Submit LOI|PRO|LIC:?|View Profile|Notes|Print|Share|Save|Report|\(\+\d+\))$/i.test(l));
+    .filter(l => !/^(Listing Contacts|Submit LOI|PRO|LIC:?|View Profile|View phone number|View email|Notes|Print|Share|Save|Report|\(\+\d+\))$/i.test(l))
+    .filter(l => !/^[*•]{3,}$/.test(l));                        // masked contact lines ("*****") on the 2026-09 detail page
   agent = (lines.find(l => PERSON_RE.test(l)) || '').trim() || null;
   license = (chunk.match(/CA\s+(\d{6,9})/) || [])[1] || null;
-  // firm = the last content line of the block that isn't the name / phone / email / license marker
-  for (let i = lines.length - 1; i >= 0; i--) {
+  // PRIMARY firm source (2026-09 page): the structured "Listed by <firm>[, <firm dba>]" line — it is present on the
+  // detail page and in the Listing Contacts block; the reverse line-scan below misread section headers
+  // ("Property Tax") and masked lines as the firm (TK-12033). Fall back to the scan only when it is absent.
+  const lbLine = ((d.lc || '') + '\n' + head).match(/Listed by\s+([^\n$]{2,120}?)(?=\s*(?:\n|\$|$))/i);
+  if (lbLine) firm = lbLine[1].split(',')[0].trim() || null;
+  // fallback: the last content line of the block that isn't the name / phone / email / license marker
+  for (let i = lines.length - 1; i >= 0 && !firm; i--) {
     const l = lines[i];
     if (l === agent) continue;
     if (/[@•]/.test(l)) continue;                     // email / masked-phone line (NOT /LIC/ — it matched firms like "Mil·lic·hap")

← 6c44ca0 CRCP data: repair 47 crexi rows minted with v2 'sales-' id p · back to Commercialrealestate · auto-data-snapshot: 2026-09-23T12:09:58 (7 data files) — dat de85550 →