`scripts/harvest-fd-mm-bb.js`

83 lines
// harvest-fd-mm-bb.js — Browserbase firm-direct DEEP-LINK harvest for Marcus & Millichap (193 deals).
// Recon (mm-recon-bb.js) revealed the real API params are camelCase pageNumber/pageSize (not the
// PascalCase the $0 attempt guessed, which the API ignored → looked page-capped). With correct params
// the flat fields (Address1/City/StateProvince/PropertyUrl) are populated, so we page the whole index
// from the site's OWN page context (session cookies, no 403), match our 193 targets by address, and take
// PropertyUrl as the firm-direct deep-link. URL-only (no roster overwrite). ~$0.04. Run:
//   NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/harvest-fd-mm-bb.js
'use strict';
const fs = require('fs');
const path = require('path');
const { chromium } = require('playwright-core');
const Browserbase = require('@browserbasehq/sdk').default;
const ROOT = path.join(__dirname, '..');
const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
function norm(a) {
  let s = String(a || '').toLowerCase().split(',')[0];
  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');
  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
}

(async () => {
  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
  const deals = ranked.ranked || ranked;
  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('marcusmillichap'))
                       .map(d => ({ id:d.id, address:d.address, key:norm(d.address) }));
  console.log(`M&M targets: ${targets.length}`);

  const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
  const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
    browserSettings: { solveCaptchas: true, viewport: { width: 1280, height: 900 } } });
  console.log('bb session', session.id);
  const browser = await chromium.connectOverCDP(session.connectUrl);
  const ctx = browser.contexts()[0];
  const page = ctx.pages()[0] || await ctx.newPage();
  await page.goto('https://www.marcusmillichap.com/properties', { waitUntil:'domcontentloaded', timeout:60000 });
  await new Promise(r=>setTimeout(r,4000));   // let the SPA establish session/cookies

  // page the whole index from the page context (correct camelCase params), keep CA rows
  const idx = await page.evaluate(async () => {
    const body = (pn) => ({ pageNumber: pn, pageSize: 100, sortOrder:'DESC', indexFieldName:'orderdate',
      facets:[], rangeFacets:[], geoFacet:{ Polygons:[], Circles:[], FieldName:'customdraw' },
      savedSearchId:null, allowedFacets:['propertytype','location','advisors','listingprice','caprate'] });
    const rows = []; let total = Infinity, pn = 1;
    while (rows.length < total && pn <= 60) {
      let j; try { const r = await fetch('/api/contentsearch/properties', { method:'POST',
        headers:{'content-type':'application/json'}, body: JSON.stringify(body(pn)) }); j = await r.json(); }
      catch (e) { break; }
      const props = (j.Results && j.Results.Properties) || []; total = (j.Results && j.Results.TotalCount) || total;
      if (!props.length) break;
      for (const p of props) {
        const href = p.PropertyUrl || ((p.Tile||'').match(/href="([^"]+)"/)||[])[1] || null;
        rows.push({ a1: p.Address1 || ((p.Tile||'').match(/class="[^"]*mm-address[^"]*">([^<]+)</)||[])[1] || null,
          city: p.City, st: p.StateProvince, url: href });
      }
      pn++;
    }
    return { total, rows };
  });
  await browser.close();
  console.log(`Paged ${idx.rows.length}/${idx.total} listings.`);

  const index = new Map();
  for (const r of idx.rows) { const k = norm(r.a1); if (k && r.url && !index.has(k)) index.set(k, r.url); }
  console.log(`Address-indexed: ${index.size}`);

  const rows = targets.map(t => { const u = index.get(t.key);
    const url = u ? (u.startsWith('http') ? u : 'https://www.marcusmillichap.com' + u) : null;
    return url ? { deal_id:t.id, address:t.address, matched:true, firm_direct_url:url, source_firm:'Marcus & Millichap' }
               : { deal_id:t.id, address:t.address, matched:false }; });
  const m = rows.filter(r=>r.matched);
  const out = { firm:'Marcus & Millichap', via:'browserbase', generated:'PENDING_STAMP', total_targets:targets.length, matched:m.length, rows };
  fs.mkdirSync(path.join(ROOT,'data','raw'), { recursive:true });
  fs.writeFileSync(path.join(ROOT,'data','raw','fd-mm-verified.json'), JSON.stringify(out,null,2));
  console.log(`\nMATCHED ${m.length}/${targets.length} deep-links → data/raw/fd-mm-verified.json`);
  console.log(JSON.stringify(m.slice(0,5).map(r=>({addr:r.address,url:r.firm_direct_url})),null,2));
  console.log('\nCOST: 1 Browserbase session ≈ $0.04');
})().catch(e => { console.error('FATAL', e.message); process.exit(1); });