crcp: wire display to prefer firm_direct_url deep-links + Browserbase Lee harvester ($0.04, 31 deep-links resolved + agent-corroborated); data write staged for approval (TK-10081)

9101210e0413be1ab00c93e26e5cf52f9c692b2a · 2026-08-01 19:38:01 -0700 · Steve Abrams
Files touched

M public/index.html
M public/mls.html
A scripts/harvest-fd-lee-bb.js
Diff

commit 9101210e0413be1ab00c93e26e5cf52f9c692b2a
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Aug 1 19:38:01 2026 -0700

    crcp: wire display to prefer firm_direct_url deep-links + Browserbase Lee harvester ($0.04, 31 deep-links resolved + agent-corroborated); data write staged for approval (TK-10081)
---
 public/index.html            |  4 +-
 public/mls.html              |  2 +
 scripts/harvest-fd-lee-bb.js | 91 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 1 deletion(-)

diff --git a/public/index.html b/public/index.html
index 693def0..6a73eea 100644
--- a/public/index.html
+++ b/public/index.html
@@ -1071,7 +1071,9 @@ const _isAgg=u=>{const h=_host(u);return !!(h&&_AGG.some(a=>h.includes(a)));};
 function listingLinkBits(p, firm){
   const bits=[];
   const listingQ='"'+(p.address||'')+'" '+(p.city||'')+' '+(firm&&firm!=='Unknown'?firm:'')+' commercial listing';
-  if(p.source&&!_isAgg(p.source)){
+  if(p.firm_direct_url&&!_isAgg(p.firm_direct_url)){   // harvested broker's-own-page deep link — highest confidence
+    bits.push(`<a class="findlink" href="${safeUrl(p.firm_direct_url)}" target="_blank" rel="noopener noreferrer">↗ View firm listing</a>`);
+  } else if(p.source&&!_isAgg(p.source)){
     bits.push(`<a class="findlink" href="${safeUrl(p.source)}" target="_blank" rel="noopener noreferrer">↗ View firm listing</a>`);
   } else {
     if(p.broker_url&&!_isAgg(p.broker_url)){
diff --git a/public/mls.html b/public/mls.html
index 47fa3bf..6764ea5 100644
--- a/public/mls.html
+++ b/public/mls.html
@@ -329,6 +329,8 @@ const _AGG=['crexi','redfin','zillow','costar','loopnet','realtor','myelisting']
 const _host=u=>{try{return new URL(u).hostname.replace(/^www\./,'');}catch{return null;}};
 const _isAgg=u=>{const h=_host(u);return !!(h&&_AGG.some(a=>h.includes(a)));};
 function brokerListing(r){
+  // 0. Harvested firm-direct deep link (broker's own property page) — highest-confidence source of truth
+  if(r.firm_direct_url && !_isAgg(r.firm_direct_url)) return {mode:'listing',href:r.firm_direct_url,label:'View firm listing ↗',title:"the broker's own property page ("+(r.firm_direct_source||'firm-direct')+")"};
   // 1. Firm-direct property listing URL → honest "view this listing"
   if(r.source && !_isAgg(r.source)) return {mode:'listing',href:r.source,label:'View firm listing ↗',title:"the broker's own property listing"};
   // 2. Aggregator / unknown source → text block with optional firm site + always a find link
diff --git a/scripts/harvest-fd-lee-bb.js b/scripts/harvest-fd-lee-bb.js
new file mode 100644
index 0000000..b74053e
--- /dev/null
+++ b/scripts/harvest-fd-lee-bb.js
@@ -0,0 +1,91 @@
+// harvest-fd-lee-bb.js — Browserbase (paid, real browser + fresh IP) firm-direct harvest for Lee.
+// Beats the plain-fetch 403 that walled the $0 attempt. Also: (1) inspects a live inventory item to
+// find the REAL listing-URL field (the $0 run wrongly grabbed link_target="_top"), and (2) adds
+// CORROBORATION — the firm-direct listing URL is trusted on address-match (it's Lee's own page for that
+// address), but the CURRENT listing agent is recorded SEPARATELY from our historical broker_agent and
+// flagged when they disagree, so we never silently overwrite with a wrong agent.
+// Writes data/raw/fd-lee-verified.json. ONE Browserbase session (~$0.04). Run:
+//   NODE_PATH=~/.claude/skills/browserbase/node_modules node scripts/harvest-fd-lee-bb.js
+'use strict';
+const fs = require('fs');
+const path = require('path');
+const { chromium } = require('playwright-core');
+const Browserbase = require('@browserbasehq/sdk').default;
+const ROOT = path.join(__dirname, '..');
+const FEED = 'https://buildout.com/plugins/9a64a93980aeae8db347e72cdfa8ca61017acc9a/inventory.json';
+
+const bbEnv = fs.readFileSync(process.env.HOME + '/.claude/skills/browserbase/.env', 'utf8');
+const env = (k) => (bbEnv.match(new RegExp('^' + k + '=(.*)$', 'm')) || [])[1]?.replace(/['"]/g, '').trim();
+
+const SUF = { street:'st', avenue:'ave', av:'ave', boulevard:'blvd', drive:'dr', road:'rd',
+  place:'pl', court:'ct', lane:'ln', terrace:'ter', parkway:'pkwy', highway:'hwy', square:'sq' };
+function norm(a) {
+  let s = String(a || '').toLowerCase().split(',')[0];
+  s = s.replace(/#\s*\S+/g, ' ').replace(/\b(ste|suite|unit|apt|no)\b.*$/,'');
+  s = s.replace(/[^\w\s]/g, ' ').replace(/\s+/g, ' ').trim();
+  return s.split(' ').map(w => SUF[w] || w).join(' ').trim();
+}
+const lastName = (n) => String(n||'').replace(/,.*$/,'').trim().split(/\s+/).pop().toLowerCase();
+const priceOf = (it) => { const p=(it.index_attributes||[]).find(x=>/price/i.test(x[0])); if(!p) return null;
+  const n=Number(String(p[1]).replace(/[^\d.]/g,'')); return Number.isFinite(n)&&n>0?n:null; };
+
+(async () => {
+  const ranked = JSON.parse(fs.readFileSync(path.join(ROOT,'data','ranked.json'),'utf8'));
+  const deals = ranked.ranked || ranked;
+  const hostOf = (u)=>{ try { return new URL(u).hostname.replace(/^www\./,''); } catch { return ''; } };
+  const targets = deals.filter(d => d.broker_url && hostOf(d.broker_url).includes('lee-associates'))
+                       .map(d => ({ id:d.id, address:d.address, our_agent:d.broker_agent||null, key:norm(d.address) }));
+  console.log(`Lee targets: ${targets.length}`);
+
+  const bb = new Browserbase({ apiKey: env('BROWSERBASE_API_KEY') });
+  const session = await bb.sessions.create({ projectId: env('BROWSERBASE_PROJECT_ID'),
+    browserSettings: { viewport: { width: 1280, height: 900 } } });
+  console.log('bb session', session.id);
+  const browser = await chromium.connectOverCDP(session.connectUrl);
+  const page = browser.contexts()[0].pages()[0] || await browser.contexts()[0].newPage();
+  const getJson = async (url) => { await page.goto(url, { waitUntil:'domcontentloaded', timeout:45000 });
+    const txt = await page.evaluate(() => document.body.innerText); return JSON.parse(txt); };
+
+  // 1) inspect field shape to find the REAL listing url
+  const first = await getJson(`${FEED}?page=1`);
+  const it0 = (first.inventory || [])[0] || {};
+  const urlFields = Object.entries(it0).filter(([k,v]) => typeof v==='string' && /https?:|\/property|\/listing|buildout\.com/i.test(v));
+  console.log('Candidate URL fields:', urlFields.map(([k,v])=>`${k}=${v}`).join(' | ') || '(none obvious)');
+  const URLKEY = (urlFields.find(([k])=>/(^|_)(url|link|permalink|show)/i.test(k)) || urlFields[0] || [])[0] || null;
+  console.log('Using URL field:', URLKEY, '| all keys:', Object.keys(it0).join(','));
+
+  // 2) page the full inventory (Browserbase IP — no 403), index by normalized address
+  const index = new Map();
+  let total = (first.meta && first.meta.total) || Infinity, seen = 0, page_i = 1, json = first;
+  while (seen < total && page_i <= 400) {
+    if (page_i > 1) { try { json = await getJson(`${FEED}?page=${page_i}`); } catch (e) { console.log(`page ${page_i} ${e.message}`); break; } }
+    const inv = json.inventory || []; if (!inv.length) break;
+    for (const it of inv) { const k = norm(it.address_one_line); if (k && !index.has(k)) index.set(k, {
+      url: URLKEY ? it[URLKEY] : null, price: priceOf(it),
+      brokers: (it.broker_contacts||[]).map(b=>({ name:b.name, email:b.email||null })).filter(b=>b.name),
+      addr: it.address_one_line }); }
+    seen += inv.length; if (page_i % 40 === 0) console.log(`  paged ${seen}/${total}`); page_i++;
+  }
+  await browser.close();
+  console.log(`Indexed ${index.size} Lee listings from ${seen} rows.`);
+
+  // 3) match + corroborate
+  const rows = targets.map(t => {
+    const hit = index.get(t.key);
+    if (!hit) return { deal_id:t.id, address:t.address, matched:false };
+    const names = hit.brokers.map(b=>b.name.replace(/,\s*CalDRE.*$/i,'').trim());
+    const agentMatch = !t.our_agent || names.some(n => lastName(n) === lastName(t.our_agent));
+    return { deal_id:t.id, address:t.address, matched:true,
+      firm_direct_url: hit.url, price: hit.price,
+      our_agent: t.our_agent, current_listing_agents: names, current_emails: hit.brokers.map(b=>b.email).filter(Boolean),
+      agent_corroborates: agentMatch, source_firm:'Lee & Associates' };
+  });
+  const m = rows.filter(r=>r.matched), corr = m.filter(r=>r.agent_corroborates);
+  const out = { firm:'Lee & Associates', via:'browserbase', url_field:URLKEY, generated:'PENDING_STAMP',
+    total_targets:targets.length, matched:m.length, agent_corroborated:corr.length, rows };
+  fs.mkdirSync(path.join(ROOT,'data','raw'), { recursive:true });
+  fs.writeFileSync(path.join(ROOT,'data','raw','fd-lee-verified.json'), JSON.stringify(out,null,2));
+  console.log(`\nMATCHED ${m.length}/${targets.length} | agent-corroborated ${corr.length}/${m.length}`);
+  console.log(JSON.stringify(m.slice(0,4).map(r=>({addr:r.address,url:r.firm_direct_url,our:r.our_agent,current:r.current_listing_agents,ok:r.agent_corroborates})),null,2));
+  console.log('\nCOST: 1 Browserbase session ≈ $0.04 → data/raw/fd-lee-verified.json');
+})().catch(e => { console.error('FATAL', e.message); process.exit(1); });
← 0914b66 auto-save: 2026-08-01T18:38:36 (4 files) — data/condos-redfi · back to Commercialrealestate · auto-save: 2026-08-01T19:38:57 (1 files) — public/deals-flow 8591e84 →