← back to Commercialrealestate
crcp: M&M firm-direct DEAD-END — API caps at 100 results server-side + no street address in feed (property-name keyed), can't match our address-based deals; $0.08 recon spent, documented (TK-10081)
5413425105b9924d799710ad57738335303bcd38 · 2026-08-01 20:15:56 -0700 · Steve Abrams
Files touched
A scripts/harvest-fd-mm-bb.jsA scripts/mm-recon-bb.js
Diff
commit 5413425105b9924d799710ad57738335303bcd38
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat Aug 1 20:15:56 2026 -0700
crcp: M&M firm-direct DEAD-END — API caps at 100 results server-side + no street address in feed (property-name keyed), can't match our address-based deals; $0.08 recon spent, documented (TK-10081)
---
scripts/harvest-fd-mm-bb.js | 82 +++++++++++++++++++++++++++++++++++++++++++++
scripts/mm-recon-bb.js | 60 +++++++++++++++++++++++++++++++++
2 files changed, 142 insertions(+)
diff --git a/scripts/harvest-fd-mm-bb.js b/scripts/harvest-fd-mm-bb.js
new file mode 100644
index 0000000..6ad816a
--- /dev/null
+++ b/scripts/harvest-fd-mm-bb.js
@@ -0,0 +1,82 @@
+// harvest-fd-mm-bb.js — Browserbase firm-direct DEEP-LINK harvest for Marcus & Millichap (193 deals).
+// Recon (mm-recon-bb.js) revealed the real API params are camelCase pageNumber/pageSize (not the
+// PascalCase the $0 attempt guessed, which the API ignored → looked page-capped). With correct params
+// the flat fields (Address1/City/StateProvince/PropertyUrl) are populated, so we page the whole index
+// from the site's OWN page context (session cookies, no 403), match our 193 targets by address, and take
+// PropertyUrl as the firm-direct deep-link. URL-only (no roster overwrite). ~$0.04. Run:
+// NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/harvest-fd-mm-bb.js
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const ROOT = path.join(__dirname, '..');
+const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
+ place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
+function norm(a) {
+ let s = String(a || '').toLowerCase().split(',')[0];
+ s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');
+ s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+ return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
+}
+
+(async () => {
+ const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
+ const deals = ranked.ranked || ranked;
+ const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
+ const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('marcusmillichap'))
+ .map(d => ({ id:d.id, address:d.address, key:norm(d.address) }));
+ console.log(`M&M targets: ${targets.length}`);
+
+ const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
+ const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
+ browserSettings: { solveCaptchas: true, viewport: { width: 1280, height: 900 } } });
+ console.log('bb session', session.id);
+ const browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+ await page.goto('https://www.marcusmillichap.com/properties', { waitUntil:'domcontentloaded', timeout:60000 });
+ await new Promise(r=>setTimeout(r,4000)); // let the SPA establish session/cookies
+
+ // page the whole index from the page context (correct camelCase params), keep CA rows
+ const idx = await page.evaluate(async () => {
+ const body = (pn) => ({ pageNumber: pn, pageSize: 100, sortOrder:'DESC', indexFieldName:'orderdate',
+ facets:[], rangeFacets:[], geoFacet:{ Polygons:[], Circles:[], FieldName:'customdraw' },
+ savedSearchId:null, allowedFacets:['propertytype','location','advisors','listingprice','caprate'] });
+ const rows = []; let total = Infinity, pn = 1;
+ while (rows.length < total && pn <= 60) {
+ let j; try { const r = await fetch('/api/contentsearch/properties', { method:'POST',
+ headers:{'content-type':'application/json'}, body: JSON.stringify(body(pn)) }); j = await r.json(); }
+ catch (e) { break; }
+ const props = (j.Results && j.Results.Properties) || []; total = (j.Results && j.Results.TotalCount) || total;
+ if (!props.length) break;
+ for (const p of props) {
+ const href = p.PropertyUrl || ((p.Tile||'').match(/href="([^"]+)"/)||[])[1] || null;
+ rows.push({ a1: p.Address1 || ((p.Tile||'').match(/class="[^"]*mm-address[^"]*">([^<]+)</)||[])[1] || null,
+ city: p.City, st: p.StateProvince, url: href });
+ }
+ pn++;
+ }
+ return { total, rows };
+ });
+ await browser.close();
+ console.log(`Paged ${idx.rows.length}/${idx.total} listings.`);
+
+ const index = new Map();
+ for (const r of idx.rows) { const k = norm(r.a1); if (k && r.url && !index.has(k)) index.set(k, r.url); }
+ console.log(`Address-indexed: ${index.size}`);
+
+ const rows = targets.map(t => { const u = index.get(t.key);
+ const url = u ? (u.startsWith('http') ? u : 'https://www.marcusmillichap.com' + u) : null;
+ return url ? { deal_id:t.id, address:t.address, matched:true, firm_direct_url:url, source_firm:'Marcus & Millichap' }
+ : { deal_id:t.id, address:t.address, matched:false }; });
+ const m = rows.filter(r=>r.matched);
+ const out = { firm:'Marcus & Millichap', via:'browserbase', generated:'PENDING_STAMP', total_targets:targets.length, matched:m.length, rows };
+ fs.mkdirSync(path.join(ROOT,'data','raw'), { recursive:true });
+ fs.writeFileSync(path.join(ROOT,'data','raw','fd-mm-verified.json'), JSON.stringify(out,null,2));
+ console.log(`\nMATCHED ${m.length}/${targets.length} deep-links → data/raw/fd-mm-verified.json`);
+ console.log(JSON.stringify(m.slice(0,5).map(r=>({addr:r.address,url:r.firm_direct_url})),null,2));
+ console.log('\nCOST: 1 Browserbase session ≈ $0.04');
+})().catch(e => { console.error('FATAL', e.message); process.exit(1); });
diff --git a/scripts/mm-recon-bb.js b/scripts/mm-recon-bb.js
new file mode 100644
index 0000000..d8a19f1
--- /dev/null
+++ b/scripts/mm-recon-bb.js
@@ -0,0 +1,60 @@
+// mm-recon-bb.js — Browserbase recon of Marcus & Millichap's property search (TK-10081 Phase 2).
+// Goal: learn how the live search scopes to a location + where the listing URL/address/brokers live,
+// so a harvester can pull our 193 M&M deals' firm-direct deep-links. Intercepts every /api/ XHR the
+// SPA fires (url+method+postData+status), tries to drive the location box, and dumps a parsed sample.
+// ONE Browserbase session (~$0.04). Run: NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/mm-recon-bb.js
+'use strict';
+const fs = require('fs');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+const sleep = (ms) => new Promise(r => setTimeout(r, ms));
+
+(async () => {
+ const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
+ const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
+ browserSettings: { solveCaptchas: true, viewport: { width: 1440, height: 1000 } } });
+ console.log('bb session', session.id);
+ const browser = await chromium.connectOverCDP(session.connectUrl);
+ const ctx = browser.contexts()[0];
+ const page = ctx.pages()[0] || await ctx.newPage();
+
+ const apiCalls = [];
+ page.on('request', req => { const u = req.url();
+ if (/marcusmillichap\.com\/api\//i.test(u)) apiCalls.push({ m: req.method(), u: u.slice(0,140), body: (req.postData()||'').slice(0,300) }); });
+
+ console.log('→ loading property search…');
+ await page.goto('https://www.marcusmillichap.com/properties', { waitUntil: 'domcontentloaded', timeout: 60000 }).catch(e=>console.log('goto:', e.message));
+ await sleep(6000);
+ console.log('title:', await page.title().catch(()=>'?'));
+
+ // try to drive a location search box (best-effort — selectors unknown, so probe a few)
+ const boxSel = ['input[placeholder*="location" i]','input[placeholder*="city" i]','input[placeholder*="search" i]','input[type="search"]','.location-search input','#location'];
+ let typed = false;
+ for (const s of boxSel) { const el = await page.$(s); if (el) { try { await el.click(); await el.type('Los Angeles, CA', { delay: 60 }); await sleep(2500);
+ // press down+enter to accept first autocomplete
+ await page.keyboard.press('ArrowDown'); await sleep(400); await page.keyboard.press('Enter'); typed = true; console.log('typed location into', s); break; } catch {} } }
+ if (!typed) console.log('no location box matched the probed selectors');
+ await sleep(5000);
+
+ // in-context: hit the properties API directly (has session cookies) and dump one Tile's parse
+ const sample = await page.evaluate(async () => {
+ try {
+ const r = await fetch('/api/contentsearch/properties', { method:'POST', headers:{'content-type':'application/json'}, body: JSON.stringify({ PageSize:5, CurrentPage:1 }) });
+ const j = await r.json();
+ const props = (j.Results && j.Results.Properties) || [];
+ const tile = props[0] && props[0].Tile;
+ const href = tile && (tile.match(/href="([^"]+)"/)||[])[1];
+ const loc = tile && (tile.match(/mm-location">([^<]+)</)||[])[1];
+ return { total: j.Results && j.Results.TotalCount, sampleKeys: props[0]?Object.keys(props[0]):[], href, loc, tileLen: tile?tile.length:0 };
+ } catch (e) { return { err: e.message }; }
+ });
+
+ await browser.close();
+ console.log('\n=== API calls captured ('+apiCalls.length+') ===');
+ apiCalls.slice(0,25).forEach(c => console.log(` ${c.m} ${c.u}${c.body?' BODY='+c.body:''}`));
+ console.log('\n=== direct properties-API sample ===');
+ console.log(JSON.stringify(sample, null, 2));
+ console.log('\nCOST: 1 Browserbase session ≈ $0.04');
+})().catch(e => { console.error('FATAL', e.message); process.exit(1); });
← f97798f residential: rental-leak floor — drop sub-$25k 'sale' prices
·
back to Commercialrealestate
·
commercialrealestate: adopt href-to-deeper-data primitives ( 9cea404 →