`scripts/backfill-crexi-brokers.js`

77 lines
#!/usr/bin/env node
// backfill-crexi-brokers.js — ALL-DAY, parallel, FREE ($0) PER-BROKER phone backfill for the Crexi tail.
// The rows the firm-level pass couldn't cover each have their own crexi.com detail page (row.source),
// which carries the listing broker's name + phone. Crexi is Akamai-protected, but openclaw's REAL
// Chrome clears it — driven over CDP in OUR OWN tabs (so the concurrent Fabricut agent's active tab is
// never stolen). N tabs run in parallel. Extracted phones append to data/broker-phone-overlay.json
// keyed by broker name; serve.js overlays them. Resumable: marks each row id done in a sidecar so a
// re-run continues the tail. Slow by design (real Chrome, throttled) — meant to grind in the background.
// Usage: node scripts/backfill-crexi-brokers.js [maxRows]
'use strict';
const fs = require('fs'), path = require('path');
const NP = require('child_process').execSync('npm root -g').toString().trim();
process.env.NODE_PATH = NP; require('module').Module._initPaths();
const { chromium } = require('playwright');
const DATA = path.join(__dirname, '..', 'data');
const OVER = path.join(DATA, 'broker-phone-overlay.json');
const DONE = path.join(DATA, '.crexi-broker-done.json');       // sidecar: row ids already attempted
const CDP = 'http://127.0.0.1:18800';
const TABS = parseInt(process.env.TABS || '4', 10);
const MAX = parseInt(process.argv[2] || '3000', 10);
const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
const fmt = p => { const d = String(p||'').replace(/\D/g,'').replace(/^1(?=\d{10}$)/,''); return d.length===10 ? `(${d.slice(0,3)}) ${d.slice(3,6)}-${d.slice(6)}` : ''; };

let overlay = fs.existsSync(OVER) ? JSON.parse(fs.readFileSync(OVER,'utf8')) : {};
let doneIds = fs.existsSync(DONE) ? new Set(JSON.parse(fs.readFileSync(DONE,'utf8'))) : new Set();
const ranked = JSON.parse(fs.readFileSync(path.join(DATA,'ranked.json'),'utf8')).ranked;
const bn = r => r.broker_agent || (r.broker_agents||[])[0] || r.broker_name;
// worklist: crexi-sourced rows whose broker still has no overlay phone (name or firm) and not yet attempted
const work = ranked.filter(r => r.source && /crexi\.com/.test(r.source) && !doneIds.has(r.id)
  && !(bn(r) && overlay[norm(bn(r))]) && !(r.broker_firm && overlay['firm:'+norm(r.broker_firm)])).slice(0, MAX);

let idx = 0, done = 0, hit = 0;
const flush = () => { fs.writeFileSync(OVER, JSON.stringify(overlay)); fs.writeFileSync(DONE, JSON.stringify([...doneIds])); };

async function worker(page) {
  while (idx < work.length) {
    const r = work[idx++];
    try {
      await page.goto(r.source, { waitUntil: 'domcontentloaded', timeout: 30000 });
      await page.waitForTimeout(1500);
      const info = await page.evaluate(() => {
        const bt = document.body.innerText;
        if (/access denied|just a moment|verify you are human/i.test(bt.slice(0, 200))) return { blocked: true };
        // broker/agent name near a phone; Crexi shows "Listed By" / a broker card with tel:
        const tel = [...document.querySelectorAll('a[href^="tel:"]')].map(a => a.getAttribute('href').replace('tel:', '')).filter(Boolean);
        const nameM = bt.match(/(Listed By|Listing (Broker|Agent)|Presented By)[:\s]*([A-Z][a-zA-Z.'-]+ [A-Z][a-zA-Z.'-]+)/);
        return { phone: tel[0] || (bt.match(/\(?\d{3}\)?[ .\-]\d{3}[ .\-]\d{4}/) || [])[0] || '', name: nameM ? nameM[3] : '' };
      });
      doneIds.add(r.id);
      if (info && !info.blocked && info.phone) {
        const ph = fmt(info.phone);
        const key = info.name ? norm(info.name) : (bn(r) ? norm(bn(r)) : null);
        if (ph && key && !overlay[key]) { overlay[key] = { phone: ph, firm: r.broker_firm || '', src: 'crexi-openclaw' }; hit++; }
      }
    } catch (_) { /* leave unmarked → retry next run */ }
    if (++done % 10 === 0) { flush(); process.stdout.write(`  ${done}/${work.length} (${hit} phones)\r`); }
    await page.waitForTimeout(700);                    // throttle — polite + reduce contention
  }
}

(async () => {
  console.log(`Crexi per-broker backfill: ${work.length} rows, ${TABS} parallel openclaw tabs ($0). Background grind.`);
  const browser = await chromium.connectOverCDP(CDP);
  const ctx = browser.contexts()[0] || await browser.newContext();
  const pages = [];
  for (let i = 0; i < TABS; i++) {
    const p = await ctx.newPage();
    p.on('dialog', d => d.dismiss().catch(() => {}));   // auto-dismiss beforeunload/alert popups (crashed the run before)
    p.setDefaultTimeout(30000);
    pages.push(p);
  }
  await Promise.allSettled(pages.map(p => worker(p)));   // one worker/page dying never kills the batch
  for (const p of pages) await p.close().catch(()=>{});
  flush();
  console.log(`\nDone: ${done} pages, ${hit} broker phones added. ${work.length - done} left (re-run to continue).`);
})().catch(e => { flush(); console.error('backfill-crexi-brokers FAILED:', e.message); process.exit(1); });