CRCP: per-broker Crexi-tail phone grind — $0 openclaw CDP own-tabs, resumable (Steve: grind the tail in background)

7992c2479330a302b83cbd9cbabed2ebb0682516 · 2026-08-19 10:38:38 -0700 · Steve Abrams

- backfill-crexi-brokers.js: 4 parallel openclaw tabs fetch each crexi detail page (Akamai-cleared by real Chrome), extract listing broker name+phone -> overlay by broker name. Sidecar .crexi-broker-done.json for resume. ~80% early hit. Background grind (~2h for 1773 rows).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

A scripts/backfill-crexi-brokers.js

Diff

commit 7992c2479330a302b83cbd9cbabed2ebb0682516
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 19 10:38:38 2026 -0700

    CRCP: per-broker Crexi-tail phone grind — $0 openclaw CDP own-tabs, resumable (Steve: grind the tail in background)
    
    - backfill-crexi-brokers.js: 4 parallel openclaw tabs fetch each crexi detail page (Akamai-cleared by real Chrome), extract listing broker name+phone -> overlay by broker name. Sidecar .crexi-broker-done.json for resume. ~80% early hit. Background grind (~2h for 1773 rows).
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 scripts/backfill-crexi-brokers.js | 70 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/scripts/backfill-crexi-brokers.js b/scripts/backfill-crexi-brokers.js
new file mode 100644
index 0000000..330e314
--- /dev/null
+++ b/scripts/backfill-crexi-brokers.js
@@ -0,0 +1,70 @@
+#!/usr/bin/env node
+// backfill-crexi-brokers.js — ALL-DAY, parallel, FREE ($0) PER-BROKER phone backfill for the Crexi tail.
+// The rows the firm-level pass couldn't cover each have their own crexi.com detail page (row.source),
+// which carries the listing broker's name + phone. Crexi is Akamai-protected, but openclaw's REAL
+// Chrome clears it — driven over CDP in OUR OWN tabs (so the concurrent Fabricut agent's active tab is
+// never stolen). N tabs run in parallel. Extracted phones append to data/broker-phone-overlay.json
+// keyed by broker name; serve.js overlays them. Resumable: marks each row id done in a sidecar so a
+// re-run continues the tail. Slow by design (real Chrome, throttled) — meant to grind in the background.
+// Usage: node scripts/backfill-crexi-brokers.js [maxRows]
+'use strict';
+const fs = require('fs'), path = require('path');
+const NP = require('child_process').execSync('npm root -g').toString().trim();
+process.env.NODE_PATH = NP; require('module').Module._initPaths();
+const { chromium } = require('playwright');
+const DATA = path.join(__dirname, '..', 'data');
+const OVER = path.join(DATA, 'broker-phone-overlay.json');
+const DONE = path.join(DATA, '.crexi-broker-done.json');       // sidecar: row ids already attempted
+const CDP = 'http://127.0.0.1:18800';
+const TABS = parseInt(process.env.TABS || '4', 10);
+const MAX = parseInt(process.argv[2] || '3000', 10);
+const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
+const fmt = p => { const d = String(p||'').replace(/\D/g,'').replace(/^1(?=\d{10}$)/,''); return d.length===10 ? `(${d.slice(0,3)}) ${d.slice(3,6)}-${d.slice(6)}` : ''; };
+
+let overlay = fs.existsSync(OVER) ? JSON.parse(fs.readFileSync(OVER,'utf8')) : {};
+let doneIds = fs.existsSync(DONE) ? new Set(JSON.parse(fs.readFileSync(DONE,'utf8'))) : new Set();
+const ranked = JSON.parse(fs.readFileSync(path.join(DATA,'ranked.json'),'utf8')).ranked;
+const bn = r => r.broker_agent || (r.broker_agents||[])[0] || r.broker_name;
+// worklist: crexi-sourced rows whose broker still has no overlay phone (name or firm) and not yet attempted
+const work = ranked.filter(r => r.source && /crexi\.com/.test(r.source) && !doneIds.has(r.id)
+  && !(bn(r) && overlay[norm(bn(r))]) && !(r.broker_firm && overlay['firm:'+norm(r.broker_firm)])).slice(0, MAX);
+
+let idx = 0, done = 0, hit = 0;
+const flush = () => { fs.writeFileSync(OVER, JSON.stringify(overlay)); fs.writeFileSync(DONE, JSON.stringify([...doneIds])); };
+
+async function worker(page) {
+  while (idx < work.length) {
+    const r = work[idx++];
+    try {
+      await page.goto(r.source, { waitUntil: 'domcontentloaded', timeout: 30000 });
+      await page.waitForTimeout(1500);
+      const info = await page.evaluate(() => {
+        const bt = document.body.innerText;
+        if (/access denied|just a moment|verify you are human/i.test(bt.slice(0, 200))) return { blocked: true };
+        // broker/agent name near a phone; Crexi shows "Listed By" / a broker card with tel:
+        const tel = [...document.querySelectorAll('a[href^="tel:"]')].map(a => a.getAttribute('href').replace('tel:', '')).filter(Boolean);
+        const nameM = bt.match(/(Listed By|Listing (Broker|Agent)|Presented By)[:\s]*([A-Z][a-zA-Z.'-]+ [A-Z][a-zA-Z.'-]+)/);
+        return { phone: tel[0] || (bt.match(/\(?\d{3}\)?[ .\-]\d{3}[ .\-]\d{4}/) || [])[0] || '', name: nameM ? nameM[3] : '' };
+      });
+      doneIds.add(r.id);
+      if (info && !info.blocked && info.phone) {
+        const ph = fmt(info.phone);
+        const key = info.name ? norm(info.name) : (bn(r) ? norm(bn(r)) : null);
+        if (ph && key && !overlay[key]) { overlay[key] = { phone: ph, firm: r.broker_firm || '', src: 'crexi-openclaw' }; hit++; }
+      }
+    } catch (_) { /* leave unmarked → retry next run */ }
+    if (++done % 10 === 0) { flush(); process.stdout.write(`  ${done}/${work.length} (${hit} phones)\r`); }
+    await page.waitForTimeout(700);                    // throttle — polite + reduce contention
+  }
+}
+
+(async () => {
+  console.log(`Crexi per-broker backfill: ${work.length} rows, ${TABS} parallel openclaw tabs ($0). Background grind.`);
+  const browser = await chromium.connectOverCDP(CDP);
+  const ctx = browser.contexts()[0] || await browser.newContext();
+  const pages = []; for (let i = 0; i < TABS; i++) pages.push(await ctx.newPage());
+  await Promise.all(pages.map(p => worker(p)));
+  for (const p of pages) await p.close().catch(()=>{});
+  flush();
+  console.log(`\nDone: ${done} pages, ${hit} broker phones added. ${work.length - done} left (re-run to continue).`);
+})().catch(e => { flush(); console.error('backfill-crexi-brokers FAILED:', e.message); process.exit(1); });

← a04fb2b Fold 8,796 new active broker-book listings into catalog (nat · back to Commercialrealestate · CRCP: wire per-broker crexi grind into daily sync (capped 40 edd6422 →