`scripts/backfill-showcase-brokers.js`

70 lines
#!/usr/bin/env node
// backfill-showcase-brokers.js — FREE ($0) broker backfill for the Showcase listings.
// The Showcase list cards carry no broker; each detail page does — in .brokerInfo-wrapper:
// "<Agent Name><Firm>...CALL <phone>". Showcase detail pages are Akamai-blocked to curl/headless,
// but openclaw's REAL Chrome clears them at $0 (vs ~$0.10-0.20/page on browserbase = $44-88 for 440).
// We connect Playwright to openclaw's Chrome over CDP and open OUR OWN tab, so we don't fight whatever
// other agent is using the shared browser's active tab. Extraction is structured DOM (no LLM cost;
// exo/local is available as a fallback but not needed). RESUMABLE: only fetches listings still missing
// a broker; writes progress back to data/showcase-listings.json after each page.
//
// Usage: node scripts/backfill-showcase-brokers.js [limit]   (limit = max pages this run, default 60)
'use strict';
const fs = require('fs');
const path = require('path');
const NP = require('child_process').execSync('npm root -g').toString().trim();
process.env.NODE_PATH = NP;
require('module').Module._initPaths();
const { chromium } = require('playwright');

const FILE = path.join(__dirname, '..', 'data', 'showcase-listings.json');
const CDP = 'http://127.0.0.1:18800';   // openclaw real-Chrome CDP endpoint (openclaw browser status)
const LIMIT = parseInt(process.argv[2] || '60', 10);

// area-code sanity: real CA local line, not a toll-free/CoStar footer number.
const TOLLFREE = new Set(['800','833','844','855','866','877','888']);
const cleanPhone = p => { const d = String(p||'').replace(/\D/g,'').replace(/^1(?=\d{10}$)/,''); return d.length===10 ? `(${d.slice(0,3)}) ${d.slice(3,6)}-${d.slice(6)}` : ''; };

(async () => {
  const doc = JSON.parse(fs.readFileSync(FILE, 'utf8'));
  const todo = doc.listings.filter(r => r.source && !r.broker_phone && !r._brokerBackfilled).slice(0, LIMIT);
  if (!todo.length) { console.log('Showcase broker backfill: nothing to do (all backfilled).'); return; }
  console.log(`Showcase broker backfill: ${todo.length} listings this run (of ${doc.listings.length} total), via openclaw ($0).`);

  const browser = await chromium.connectOverCDP(CDP);
  const ctx = browser.contexts()[0] || await browser.newContext();
  const page = await ctx.newPage();                 // OUR OWN tab — doesn't disturb other agents' active tab
  let done = 0, hit = 0;
  for (const r of todo) {
    try {
      await page.goto(r.source, { waitUntil: 'domcontentloaded', timeout: 30000 });
      await page.waitForTimeout(1200);
      const info = await page.evaluate(() => {
        const el = document.querySelector('.brokerInfo-wrapper') || document.querySelector('.brokerInfo-component');
        if (!el) return null;
        const t = (el.innerText || '').replace(/\s+/g, ' ').trim();
        const phone = (t.match(/CALL\s*([0-9()\-. ]{10,16})/i) || [])[1] || (t.match(/\(?\d{3}\)?[ .\-]\d{3}[ .\-]\d{4}/) || [])[0] || '';
        // "Andrew Sparks The Sparks Agency More Info CALL ..." → name = up to firm; firm heuristic: the run before 'More Info'/'CALL'
        const head = t.split(/More Info|CALL/i)[0].trim();
        return { head, phone };
      });
      r._brokerBackfilled = true;                    // mark attempted (resumable) even if no broker found
      if (info && info.head) {
        // head ≈ "<Agent Name><Firm>" concatenated; split on the case boundary between name and firm is unreliable,
        // so store the whole head as broker label + the phone; usre still resolves via the firm/name text.
        r.broker_label = info.head.slice(0, 80);
        const ph = cleanPhone(info.phone);
        if (ph && !TOLLFREE.has(ph.slice(1,4))) { r.broker_phone = ph; hit++; }
        r.broker_agents = r.broker_agents && r.broker_agents.length ? r.broker_agents : [info.head.slice(0, 50)];
      }
    } catch (e) { r._brokerBackfilled = true; /* mark so we don't spin on a dead page */ }
    done++;
    if (done % 10 === 0) { fs.writeFileSync(FILE, JSON.stringify(doc)); process.stdout.write(`  ${done}/${todo.length} (${hit} phones)\r`); }
    await page.waitForTimeout(600);                  // throttle — be polite + reduce contention
  }
  await page.close();
  fs.writeFileSync(FILE, JSON.stringify(doc));
  const remaining = doc.listings.filter(r => r.source && !r._brokerBackfilled).length;
  console.log(`\nDone: ${done} pages, ${hit} real phones backfilled. ${remaining} listings still pending (re-run to continue).`);
})().catch(e => { console.error('backfill FAILED:', e.message); process.exit(1); });