`scripts/harvest-fd-lee-bb.js`

92 lines
// harvest-fd-lee-bb.js — Browserbase (paid, real browser + fresh IP) firm-direct harvest for Lee.
// Beats the plain-fetch 403 that walled the $0 attempt. Also: (1) inspects a live inventory item to
// find the REAL listing-URL field (the $0 run wrongly grabbed link_target="_top"), and (2) adds
// CORROBORATION — the firm-direct listing URL is trusted on address-match (it's Lee's own page for that
// address), but the CURRENT listing agent is recorded SEPARATELY from our historical broker_agent and
// flagged when they disagree, so we never silently overwrite with a wrong agent.
// Writes data/raw/fd-lee-verified.json. ONE Browserbase session (~$0.04). Run:
//   NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/harvest-fd-lee-bb.js
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const ROOT = path.join(__dirname, '..');
const FEED = 'https://buildout.com/plugins/9a64a93980aeae8db347e72cdfa8ca61017acc9a/inventory.json';

const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();

const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
function norm(a) {
  let s = String(a || '').toLowerCase().split(',')[0];
  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');
  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
}
const lastName = (n) => String(n||'').replace(/,.*$/,'').trim().split(/\s+/).pop().toLowerCase();
const priceOf = (it) => { const p=(it.index_attributes||[]).find(x=>/price/i.test(x[0])); if(!p) return null;
  const n=Number(String(p[1]).replace(/[^\d.]/g,'')); return Number.isFinite(n)&&n>0?n:null; };

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
  const deals = ranked.ranked || ranked;
  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lee-associates'))
                       .map(d => ({ id:d.id, address:d.address, our_agent:d.broker_agent||null, key:norm(d.address) }));
  console.log(`Lee targets: ${targets.length}`);

  const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
  const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
    browserSettings: { viewport: { width: 1280, height: 900 } } });
  console.log('bb session', session.id);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();
  const getJson = async (url) => { await page.goto(url, { waitUntil:'domcontentloaded', timeout:45000 });
    const txt = await page.evaluate(() => document.body.innerText); return JSON.parse(txt); };

  // 1) inspect field shape to find the REAL listing url
  const first = await getJson(`${FEED}?page=1`);
  const it0 = (first.inventory || [])[0] || {};
  const urlFields = Object.entries(it0).filter(([k,v]) => typeof v==='string' && /https?:|\/property|\/listing|buildout\.com/i.test(v));
  console.log('Candidate URL fields:', urlFields.map(([k,v])=>`${k}=${v}`).join(' | ') || '(none obvious)');
  const URLKEY = (urlFields.find(([k])=>/(^|_)(url|link|permalink|show)/i.test(k)) || urlFields[0] || [])[0] || null;
  console.log('Using URL field:', URLKEY, '| all keys:', Object.keys(it0).join(','));

  // 2) page the full inventory (Browserbase IP — no 403), index by normalized address
  const index = new Map();
  let total = (first.meta && first.meta.total) || Infinity, seen = 0, page_i = 1, json = first;
  while (seen < total && page_i <= 400) {
    if (page_i > 1) { try { json = await getJson(`${FEED}?page=${page_i}`); } catch (e) { console.log(`page ${page_i} ${e.message}`); break; } }
    const inv = json.inventory || []; if (!inv.length) break;
    for (const it of inv) { const k = norm(it.address_one_line); if (k && !index.has(k)) index.set(k, {
      url: URLKEY ? it[URLKEY] : null, price: priceOf(it),
      brokers: (it.broker_contacts||[]).map(b=>({ name:b.name, email:b.email||null })).filter(b=>b.name),
      addr: it.address_one_line }); }
    seen += inv.length; if (page_i % 40 === 0) console.log(`  paged ${seen}/${total}`); page_i++;
  }
  await browser.close();
  console.log(`Indexed ${index.size} Lee listings from ${seen} rows.`);

  // 3) match + corroborate
  const rows = targets.map(t => {
    const hit = index.get(t.key);
    if (!hit) return { deal_id:t.id, address:t.address, matched:false };
    const names = hit.brokers.map(b=>b.name.replace(/,\s*CalDRE.*$/i,'').trim());
    const agentMatch = !t.our_agent || names.some(n => lastName(n) === lastName(t.our_agent));
    return { deal_id:t.id, address:t.address, matched:true,
      firm_direct_url: hit.url, price: hit.price,
      our_agent: t.our_agent, current_listing_agents: names, current_emails: hit.brokers.map(b=>b.email).filter(Boolean),
      agent_corroborates: agentMatch, source_firm:'Lee & Associates' };
  });
  const m = rows.filter(r=>r.matched), corr = m.filter(r=>r.agent_corroborates);
  const out = { firm:'Lee & Associates', via:'browserbase', url_field:URLKEY, generated:'PENDING_STAMP',
    total_targets:targets.length, matched:m.length, agent_corroborated:corr.length, rows };
  fs.mkdirSync(path.join(ROOT,'data','raw'), { recursive:true });
  fs.writeFileSync(path.join(ROOT,'data','raw','fd-lee-verified.json'), JSON.stringify(out,null,2));
  console.log(`\nMATCHED ${m.length}/${targets.length} | agent-corroborated ${corr.length}/${m.length}`);
  console.log(JSON.stringify(m.slice(0,4).map(r=>({addr:r.address,url:r.firm_direct_url,our:r.our_agent,current:r.current_listing_agents,ok:r.agent_corroborates})),null,2));
  console.log('\nCOST: 1 Browserbase session ≈ $0.04 → data/raw/fd-lee-verified.json');
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });