crcp: Matthews firm-direct recon — no listings JSON XHR (SSR/Next.js, data in __NEXT_DATA__); no cheap crack, deferred (TK-10081)

a04788cbc023d92292279ffe7f26a79c2e8440d3 · 2026-08-01 20:55:54 -0700 · Steve Abrams
Files touched

A scripts/harvest-fd-matthews-bb.js
Diff

commit a04788cbc023d92292279ffe7f26a79c2e8440d3
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Aug 1 20:55:54 2026 -0700

    crcp: Matthews firm-direct recon — no listings JSON XHR (SSR/Next.js, data in __NEXT_DATA__); no cheap crack, deferred (TK-10081)
---
 scripts/harvest-fd-matthews-bb.js | 86 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/scripts/harvest-fd-matthews-bb.js b/scripts/harvest-fd-matthews-bb.js
new file mode 100644
index 0000000..ec6f001
--- /dev/null
+++ b/scripts/harvest-fd-matthews-bb.js
@@ -0,0 +1,86 @@
+// harvest-fd-matthews-bb.js — Browserbase recon+harvest for Matthews (29 deals). Platform unknown, so
+// ADAPTIVE: capture every JSON XHR the listings search fires, auto-detect the one that's an array of
+// listing objects exposing a street ADDRESS + a listing URL (the M&M dead-end was no-street-address), and
+// if found, match our 29 targets → firm_direct_url. Fails fast with a shape dump if addresses are hidden.
+// ~$0.04. Run: NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/harvest-fd-matthews-bb.js
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const ROOT = path.join(__dirname, '..');
+const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
+  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
+function norm(a) { let s = String(a||'').toLowerCase().split(',')[0];
+  s = s.replace(/#\s*\S+/g,' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'').replace(/[^\w\s]/g,' ').replace(/\s+/g,' ').trim();
+  return s.split(' ').map(w=>SUF[w]||w).join(' ').trim(); }
+// find the deepest array of objects inside an arbitrary JSON
+function findArrays(o, out=[], depth=0) { if (depth>6||!o||typeof o!=='object') return out;
+  if (Array.isArray(o)) { if (o.length && typeof o[0]==='object') out.push(o); return out; }
+  for (const v of Object.values(o)) findArrays(v, out, depth+1); return out; }
+const ADDR_RE = /address|street|addr1|line1/i, URL_RE = /url|link|slug|permalink|href|detail/i;
+const strOf = (obj, re) => { for (const [k,v] of Object.entries(obj)) if (re.test(k) && typeof v==='string' && v.trim()) return v; return null; };
+
+(async () => {
+  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
+  const deals = ranked.ranked || ranked;
+  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
+  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('matthews'))
+                       .map(d => ({ id:d.id, address:d.address, key:norm(d.address) }));
+  console.log(`Matthews targets: ${targets.length}`);
+
+  const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
+  const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
+    browserSettings: { solveCaptchas: true, viewport: { width:1280, height:900 } } });
+  console.log('bb session', session.id);
+  const browser = await chromium.connectOverCDP(session.connectUrl);
+  const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();
+
+  const jsonResponses = [];
+  page.on('response', async res => { const u = res.url();
+    if (!/matthews\.com/i.test(u)) return;
+    const ct = res.headers()['content-type']||''; if (!/json/i.test(ct)) return;
+    try { const j = await res.json(); jsonResponses.push({ u: u.slice(0,120), j }); } catch {} });
+
+  for (const url of ['https://www.matthews.com/listings/','https://www.matthews.com/properties/','https://www.matthews.com/listings/?type=sale']) {
+    try { await page.goto(url, { waitUntil:'networkidle', timeout:45000 }); break; } catch (e) { console.log('goto', url, e.message); }
+  }
+  await new Promise(r=>setTimeout(r,5000));
+  // scroll to trigger lazy XHRs
+  await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight)).catch(()=>{});
+  await new Promise(r=>setTimeout(r,3000));
+  await browser.close();
+
+  console.log(`\nJSON responses captured: ${jsonResponses.length}`);
+  // find candidate listing arrays that expose an address + url
+  let best = null;
+  for (const {u,j} of jsonResponses) for (const arr of findArrays(j)) {
+    const withAddr = arr.filter(o => strOf(o, ADDR_RE)).length;
+    if (withAddr >= Math.min(3, arr.length) && arr.length >= 3) {
+      const sample = arr[0];
+      const cand = { u, len: arr.length, addrKey: Object.keys(sample).find(k=>ADDR_RE.test(k)&&typeof sample[k]==='string'),
+        urlKey: Object.keys(sample).find(k=>URL_RE.test(k)&&typeof sample[k]==='string'), arr };
+      if (!best || cand.len > best.len) best = cand;
+    }
+  }
+  if (!best) {
+    console.log('NO listing array with street addresses found. Endpoints seen:');
+    jsonResponses.forEach(r=>console.log('  '+r.u));
+    console.log('\nVERDICT: Matthews does not expose street addresses in captured JSON → likely dead-end (like M&M). $0.04 recon.');
+    return;
+  }
+  console.log(`Listing array: ${best.len} items · addrKey=${best.addrKey} urlKey=${best.urlKey} · ${best.u}`);
+  console.log('sample:', JSON.stringify({ addr: best.arr[0][best.addrKey], url: best.urlKey?best.arr[0][best.urlKey]:'(none)' }));
+  const index = new Map();
+  for (const o of best.arr) { const k = norm(o[best.addrKey]); const url = best.urlKey?o[best.urlKey]:null; if (k && url && !index.has(k)) index.set(k, url); }
+  const rows = targets.map(t => { let u = index.get(t.key);
+    if (u && !u.startsWith('http')) u = 'https://www.matthews.com' + (u.startsWith('/')?'':'/') + u;
+    return u ? { deal_id:t.id, address:t.address, matched:true, firm_direct_url:u, source_firm:'Matthews' } : { deal_id:t.id, address:t.address, matched:false }; });
+  const m = rows.filter(r=>r.matched);
+  fs.writeFileSync(path.join(ROOT,'data','raw','fd-matthews-verified.json'), JSON.stringify({ firm:'Matthews', via:'browserbase', total_targets:targets.length, matched:m.length, rows }, null, 2));
+  console.log(`\nMATCHED ${m.length}/${targets.length} → data/raw/fd-matthews-verified.json`);
+  console.log(JSON.stringify(m.slice(0,5).map(r=>({addr:r.address,url:r.firm_direct_url})),null,2));
+  console.log('\nCOST: 1 Browserbase session ≈ $0.04');
+})().catch(e => { console.error('FATAL', e.message); process.exit(1); });
← a241df1 condos: normalize junk broker/firm placeholders (N/A etc) to · back to Commercialrealestate · chore: lint (node --check ✓) + v0.15.0 (firm-direct deep-lin 03f0159 →