`scripts/backfill-crexi-firms.js`

65 lines
#!/usr/bin/env node
// backfill-crexi-firms.js — ALL-DAY, parallel, FREE ($0) firm-phone backfill for the Crexi-base rows.
// The ~2,400 usre-miss rows cluster into ~1,248 firms (branch-specific: "Marcus & Millichap - Encino"
// etc.), and a firm's local office line is a legitimate contact. This resolves each firm's phone via a
// Google search in openclaw's REAL Chrome (clears anti-bot, logged-in, $0) — running N tabs IN PARALLEL
// (Steve: "max-it all day... parallel in browsers if needed... just go"). Appends to the shared
// data/broker-phone-overlay.json under 'firm:<normfirm>'; serve.js overlays it onto every row at that
// firm lacking a phone. Resumable (skips firms already resolved or marked no-hit).
'use strict';
const fs = require('fs'), path = require('path');
const NP = require('child_process').execSync('npm root -g').toString().trim();
process.env.NODE_PATH = NP; require('module').Module._initPaths();
const { chromium } = require('playwright');
const DATA = path.join(__dirname, '..', 'data');
const OVER = path.join(DATA, 'broker-phone-overlay.json');
const CDP = 'http://127.0.0.1:18800';
const TABS = parseInt(process.env.TABS || '4', 10);         // parallel tabs in openclaw's Chrome
const CA_AREA = new Set(['209','213','279','310','323','341','350','408','415','424','442','510','530','559','562','619','626','628','650','657','661','669','707','714','747','760','805','818','820','831','858','909','916','925','935','949','951']);
const norm = s => String(s || '').toLowerCase().replace(/[^a-z0-9]/g, '');
const area = p => { const d = String(p||'').replace(/\D/g,'').replace(/^1(?=\d{10}$)/,''); return d.slice(0,3); };
const fmt = p => { const d = String(p||'').replace(/\D/g,'').replace(/^1(?=\d{10}$)/,''); return d.length===10 ? `(${d.slice(0,3)}) ${d.slice(3,6)}-${d.slice(6)}` : ''; };

const firms = JSON.parse(fs.readFileSync('/tmp/miss-firms.json', 'utf8'));
let overlay = fs.existsSync(OVER) ? JSON.parse(fs.readFileSync(OVER, 'utf8')) : {};
const queue = firms.filter(f => f && !overlay['firm:' + norm(f)] && !overlay['firm:' + norm(f) + ':nohit']);
let idx = 0, done = 0, hit = 0;
const saveOverlay = () => fs.writeFileSync(OVER, JSON.stringify(overlay));

async function worker(page, wid) {
  while (idx < queue.length) {
    const firm = queue[idx++];
    const key = 'firm:' + norm(firm);
    try {
      const q = encodeURIComponent(firm + ' commercial real estate Los Angeles phone number');
      await page.goto('https://www.google.com/search?q=' + q, { waitUntil: 'domcontentloaded', timeout: 25000 });
      await page.waitForTimeout(700 + Math.floor((wid + idx) % 5) * 120);
      const phone = await page.evaluate(() => {
        const t = document.body.innerText;
        // prefer the knowledge-panel "Phone:" line, else first CA-looking number
        const kp = (t.match(/Phone:\s*(\(?\d{3}\)?[ .\-]\d{3}[ .\-]\d{4})/i) || [])[1];
        if (kp) return kp;
        const all = t.match(/\(?\d{3}\)?[ .\-]\d{3}[ .\-]\d{4}/g) || [];
        return all[0] || '';
      });
      const ph = fmt(phone);
      if (ph && CA_AREA.has(area(ph))) { overlay[key] = { phone: ph, firm, tier: 'firm', src: 'google-openclaw' }; hit++; }
      else { overlay[key + ':nohit'] = 1; }          // mark so we don't re-try a dry firm
    } catch (_) { /* transient — leave unmarked to retry next run */ }
    if (++done % 8 === 0) { saveOverlay(); process.stdout.write(`  ${done}/${queue.length} firms (${hit} phones)\r`); }
    await page.waitForTimeout(400);
  }
}

(async () => {
  console.log(`Crexi firm-phone backfill: ${queue.length} firms, ${TABS} parallel openclaw tabs ($0). All-day.`);
  const browser = await chromium.connectOverCDP(CDP);
  const ctx = browser.contexts()[0] || await browser.newContext();
  const pages = [];
  for (let i = 0; i < TABS; i++) pages.push(await ctx.newPage());
  await Promise.all(pages.map((p, i) => worker(p, i)));
  for (const p of pages) await p.close().catch(() => {});
  saveOverlay();
  console.log(`\nDone: ${done} firms resolved, ${hit} firm phones added to overlay.`);
})().catch(e => { saveOverlay(); console.error('backfill-crexi-firms FAILED:', e.message); process.exit(1); });