`scripts/harvest-fd-matthews-bb.js`

87 lines
// harvest-fd-matthews-bb.js — Browserbase recon+harvest for Matthews (29 deals). Platform unknown, so
// ADAPTIVE: capture every JSON XHR the listings search fires, auto-detect the one that's an array of
// listing objects exposing a street ADDRESS + a listing URL (the M&M dead-end was no-street-address), and
// if found, match our 29 targets → firm_direct_url. Fails fast with a shape dump if addresses are hidden.
// ~$0.04. Run: NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/harvest-fd-matthews-bb.js
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
function norm(a) { let s = String(a||'').toLowerCase().split(',')[0];
  s = s.replace(/#\s*\S+/g,' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'').replace(/[^\w\s]/g,' ').replace(/\s+/g,' ').trim();
  return s.split(' ').map(w=>SUF[w]||w).join(' ').trim(); }
// find the deepest array of objects inside an arbitrary JSON
function findArrays(o, out=[], depth=0) { if (depth>6||!o||typeof o!=='object') return out;
  if (Array.isArray(o)) { if (o.length && typeof o[0]==='object') out.push(o); return out; }
  for (const v of Object.values(o)) findArrays(v, out, depth+1); return out; }
const ADDR_RE = /address|street|addr1|line1/i, URL_RE = /url|link|slug|permalink|href|detail/i;
const strOf = (obj, re) => { for (const [k,v] of Object.entries(obj)) if (re.test(k) && typeof v==='string' && v.trim()) return v; return null; };

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
  const deals = ranked.ranked || ranked;
  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('matthews'))
                       .map(d => ({ id:d.id, address:d.address, key:norm(d.address) }));
  console.log(`Matthews targets: ${targets.length}`);

  const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
  const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
    browserSettings: { solveCaptchas: true, viewport: { width:1280, height:900 } } });
  console.log('bb session', session.id);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();

  const jsonResponses = [];
  page.on('response', async res => { const u = res.url();
    if (!/matthews\.com/i.test(u)) return;
    const ct = res.headers()['content-type']||''; if (!/json/i.test(ct)) return;
    try { const j = await res.json(); jsonResponses.push({ u: u.slice(0,120), j }); } catch {} });

  for (const url of ['https://www.matthews.com/listings/','https://www.matthews.com/properties/','https://www.matthews.com/listings/?type=sale']) {
    try { await page.goto(url, { waitUntil:'networkidle', timeout:45000 }); break; } catch (e) { console.log('goto', url, e.message); }
  }
  await new Promise(r=>setTimeout(r,5000));
  // scroll to trigger lazy XHRs
  await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(()=>{});
  await new Promise(r=>setTimeout(r,3000));
  await browser.close();

  console.log(`\nJSON responses captured: ${jsonResponses.length}`);
  // find candidate listing arrays that expose an address + url
  let best = null;
  for (const {u,j} of jsonResponses) for (const arr of findArrays(j)) {
    const withAddr = arr.filter(o => strOf(o, ADDR_RE)).length;
    if (withAddr >= Math.min(3, arr.length) && arr.length >= 3) {
      const sample = arr[0];
      const cand = { u, len: arr.length, addrKey: Object.keys(sample).find(k=>ADDR_RE.test(k)&&typeof sample[k]==='string'),
        urlKey: Object.keys(sample).find(k=>URL_RE.test(k)&&typeof sample[k]==='string'), arr };
      if (!best || cand.len > best.len) best = cand;
    }
  }
  if (!best) {
    console.log('NO listing array with street addresses found. Endpoints seen:');
    jsonResponses.forEach(r=>console.log('  '+r.u));
    console.log('\nVERDICT: Matthews does not expose street addresses in captured JSON → likely dead-end (like M&M). $0.04 recon.');
    return;
  }
  console.log(`Listing array: ${best.len} items · addrKey=${best.addrKey} urlKey=${best.urlKey} · ${best.u}`);
  console.log('sample:', JSON.stringify({ addr: best.arr[0][best.addrKey], url: best.urlKey?best.arr[0][best.urlKey]:'(none)' }));
  const index = new Map();
  for (const o of best.arr) { const k = norm(o[best.addrKey]); const url = best.urlKey?o[best.urlKey]:null; if (k && url && !index.has(k)) index.set(k, url); }
  const rows = targets.map(t => { let u = index.get(t.key);
    if (u && !u.startsWith('http')) u = 'https://www.matthews.com' + (u.startsWith('/')?'':'/') + u;
    return u ? { deal_id:t.id, address:t.address, matched:true, firm_direct_url:u, source_firm:'Matthews' } : { deal_id:t.id, address:t.address, matched:false }; });
  const m = rows.filter(r=>r.matched);
  fs.writeFileSync(path.join(ROOT,'data','raw','fd-matthews-verified.json'), JSON.stringify({ firm:'Matthews', via:'browserbase', total_targets:targets.length, matched:m.length, rows }, null, 2));
  console.log(`\nMATCHED ${m.length}/${targets.length} → data/raw/fd-matthews-verified.json`);
  console.log(JSON.stringify(m.slice(0,5).map(r=>({addr:r.address,url:r.firm_direct_url})),null,2));
  console.log('\nCOST: 1 Browserbase session ≈ $0.04');
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });