← back to Commercialrealestate
condos both-sides: buyer-side scraper (Redfin buyingAgents) + /api/condos overlay plumbing — TK-10243
2bee9321d461c236870a3eb175868575d05770b3 · 2026-08-05 12:04:18 -0700 · Steve Abrams
enrich-condo-brokers.js: --buyer-side mode targets SOLD listings, extractBuyingAgents
parses Redfin mainHouseInfo.buyingAgents (parallel to listingAgents), hooks buyer DRE#
into the full CA record, merge-preserves prior buyer side. serve.js overlayBroker now
passes buyer_agents + agent_email through to /api/condos. Buyer-side parser unit-tested
offline ($0): captures name/firm/DRE/phone, empty [] → no buyer. Live scrape is metered
(Browserbase), run separately with cost caps.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M scripts/enrich-condo-brokers.jsM scripts/serve.js
Diff
commit 2bee9321d461c236870a3eb175868575d05770b3
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Aug 5 12:04:18 2026 -0700
condos both-sides: buyer-side scraper (Redfin buyingAgents) + /api/condos overlay plumbing — TK-10243
enrich-condo-brokers.js: --buyer-side mode targets SOLD listings, extractBuyingAgents
parses Redfin mainHouseInfo.buyingAgents (parallel to listingAgents), hooks buyer DRE#
into the full CA record, merge-preserves prior buyer side. serve.js overlayBroker now
passes buyer_agents + agent_email through to /api/condos. Buyer-side parser unit-tested
offline ($0): captures name/firm/DRE/phone, empty [] → no buyer. Live scrape is metered
(Browserbase), run separately with cost caps.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
scripts/enrich-condo-brokers.js | 47 ++++++++++++++++++++++++++++++++++-------
scripts/serve.js | 6 ++++++
2 files changed, 45 insertions(+), 8 deletions(-)
diff --git a/scripts/enrich-condo-brokers.js b/scripts/enrich-condo-brokers.js
index c559efe..8ab47ef 100644
--- a/scripts/enrich-condo-brokers.js
+++ b/scripts/enrich-condo-brokers.js
@@ -40,6 +40,9 @@ const arg = (k, d) => { const i = process.argv.indexOf(k); return i > -1 ? (proc
const LIMIT = +arg('--limit', 0) || Infinity;
const DELAY = +arg('--delay', 1800);
const ONLY_MISSING = process.argv.includes('--only-missing');
+// Buyer-SIDE mode (TK-10243): target SOLD listings (where a buyer's agent can exist) instead of the
+// fha_expired set, to capture "both sides" of the deal. Listing side is still refreshed on the pass.
+const BUYER_SIDE = process.argv.includes('--buyer-side') || process.argv.includes('--sold');
const USE_BROWSER = process.argv.includes('--browser'); // real Chrome via Playwright ($0, but Redfin WAF blocks flagged IPs)
const USE_BB = process.argv.includes('--browserbase'); // metered fallback: cloud stealth browser + proxies (~$0.15/session-min)
const ONLY_IDS = (arg('--ids', '') || '').split(',').filter(Boolean);
@@ -52,8 +55,13 @@ const jload = (p, d) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); }
// ---- Redfin listing-agent extraction (embedded JSON in the page HTML) ----
// Pull every {name, firm, dre} from the (unescaped) listingAgents array. Prefers a real JSON.parse
// of the balanced-bracket array (nesting-safe), falls back to a per-object regex, dedups by dre||name.
-function extractAgents(html) {
- const key = '"listingAgents":';
+function extractAgents(html) { return extractAgentArray(html, 'listingAgents'); }
+// Buyer-SIDE agents on a SOLD listing (Redfin key mainHouseInfo.buyingAgents — parallel to
+// listingAgents). Often empty ("[]") because Redfin doesn't always publish the buyer's agent; when
+// present it carries the same {agentName, brokerName, license} shape. TK-10243.
+function extractBuyingAgents(html) { return extractAgentArray(html, 'buyingAgents'); }
+function extractAgentArray(html, keyName) {
+ const key = '"' + keyName + '":';
const ki = html.indexOf(key);
if (ki < 0) return [];
const start = html.indexOf('[', ki);
@@ -116,6 +124,16 @@ function extractBroker(raw) {
if (!out.broker_name) { const m = html.match(/"listingAgentName":"([^"]{1,80})"/); if (m && m[1].trim()) out.broker_name = m[1].trim(); }
const ap = html.match(/"listingAgentNumber":"([0-9()\-\s.]{7,20})"/); if (ap) out.agent_phone = ap[1].trim();
const op = html.match(/"listingBrokerNumber":"([0-9()\-\s.]{7,20})"/); if (op) out.office_phone = op[1].trim();
+ // BUYER SIDE (sold listings) — the "both sides" data. Captured when Redfin publishes it; blank
+ // otherwise (active listings have no buyer side yet). TK-10243.
+ const buyers = extractBuyingAgents(html);
+ if (buyers.length) {
+ out.buyer_agents = buyers;
+ if (buyers[0].name) out.buyer_agent_name = buyers[0].name;
+ if (buyers[0].firm) out.buyer_firm_name = buyers[0].firm;
+ if (buyers[0].dre) out.buyer_agent_dre = buyers[0].dre;
+ }
+ const bp = html.match(/"buyingAgentNumber":"([0-9()\-\s.]{7,20})"/); if (bp) out.buyer_agent_phone = bp[1].trim();
return out;
}
@@ -175,13 +193,20 @@ async function hookDre(dre, cache) {
async function main() {
const doc = jload(CONDOS, { condos: [] });
const exp = doc.condos.filter(c => c.warrantable_status === 'fha_expired');
+ // SOLD set for buyer-side "both sides" capture — status/market_status says sold/closed, or a sold
+ // price/date is present. Only these can carry a buyer's agent.
+ const sold = doc.condos.filter(c => /sold|closed/i.test(c.status || '') || /sold|closed/i.test(c.market_status || '') || c.sold_date || c.sold_price);
const brokers = jload(BROKERS, {});
const dreCache = jload(DRECACHE, {});
seedDreCacheFromLicensed(dreCache);
- let targets = exp;
- if (ONLY_IDS.length) targets = exp.filter(c => ONLY_IDS.includes(c.id));
- if (ONLY_MISSING) targets = targets.filter(c => !(brokers[c.id] && brokers[c.id].broker_name));
+ let targets = BUYER_SIDE ? sold : exp;
+ const baseSet = targets;
+ if (ONLY_IDS.length) targets = targets.filter(c => ONLY_IDS.includes(c.id));
+ // --only-missing means: in buyer-side mode, retry rows with no buyer_agents yet; else no broker yet.
+ if (ONLY_MISSING) targets = targets.filter(c => BUYER_SIDE
+ ? !(brokers[c.id] && Array.isArray(brokers[c.id].buyer_agents) && brokers[c.id].buyer_agents.length)
+ : !(brokers[c.id] && brokers[c.id].broker_name));
// Browser context. Local real-Chrome ($0) is blocked by Redfin's AWS-WAF on flagged IPs, so the
// metered path is Browserbase: cloud stealth browser + residential proxies + solveCaptchas that
@@ -207,7 +232,7 @@ async function main() {
console.log(' mode: real Chrome (Playwright)');
}
- console.log(`condo-broker enrich: ${targets.length} target(s) of ${exp.length} fha_expired · delay ~${DELAY}ms · $0 local`);
+ console.log(`condo-broker enrich${BUYER_SIDE ? ' [BUYER-SIDE/sold]' : ''}: ${targets.length} target(s) of ${baseSet.length} ${BUYER_SIDE ? 'sold' : 'fha_expired'} · delay ~${DELAY}ms · $0 local`);
const blocked = [];
let done = 0, gotBroker = 0, gotDre = 0;
// Cost/abort guardrails (Codex dissent): a hot Redfin WAF polls ~25s per blocked listing, so an
@@ -243,16 +268,22 @@ async function main() {
if (d) a.dre_record = d;
}
}
+ // Hook the BUYER-side license numbers into their full DRE records too (TK-10243, $0/cached).
+ if (Array.isArray(b.buyer_agents)) {
+ for (const a of b.buyer_agents) { if (!a.dre) continue; const d = await hookDre(a.dre, dreCache); if (d) a.dre_record = d; }
+ }
// NEVER clobber good data with an empty scrape (WAF blocks return no broker). Only overwrite
// when this fetch actually yielded broker/agent data; on a block, preserve any prior record and
// just flag the miss if we had nothing before (so --only-missing can retry it later).
- const gotData = b.broker_name || b.firm_name || (Array.isArray(b.agents) && b.agents.length);
+ const gotData = b.broker_name || b.firm_name || (Array.isArray(b.agents) && b.agents.length) || (Array.isArray(b.buyer_agents) && b.buyer_agents.length);
if (gotData) {
const prior = brokers[c.id] || {};
// Merge over the prior record — a PARTIAL hit (e.g. fallback listingAgentName with no agents
// array) must NOT drop the co-agent license list a previous full scrape recorded (Cody gate).
const mergedAgents = (Array.isArray(b.agents) && b.agents.length) ? b.agents : (prior.agents || undefined);
- brokers[c.id] = { ...prior, id: c.id, address: c.address, city: c.city, ...b, agents: mergedAgents, dre: dre || prior.dre || undefined, fetched_at: new Date().toISOString(), http: status };
+ // Likewise never drop a previously-captured buyer side on a pass that didn't re-find it.
+ const mergedBuyers = (Array.isArray(b.buyer_agents) && b.buyer_agents.length) ? b.buyer_agents : (prior.buyer_agents || undefined);
+ brokers[c.id] = { ...prior, id: c.id, address: c.address, city: c.city, ...b, agents: mergedAgents, buyer_agents: mergedBuyers, dre: dre || prior.dre || undefined, fetched_at: new Date().toISOString(), http: status };
} else if (!brokers[c.id] || !(brokers[c.id].broker_name || brokers[c.id].broker_dre)) {
brokers[c.id] = { id: c.id, address: c.address, city: c.city, fetched_at: new Date().toISOString(), http: status, blocked: true };
}
diff --git a/scripts/serve.js b/scripts/serve.js
index 65c3319..2bf8667 100644
--- a/scripts/serve.js
+++ b/scripts/serve.js
@@ -742,6 +742,12 @@ function overlayBroker(row) {
if (b.broker_dre) o.broker_dre = b.broker_dre;
if (b.agent_phone) o.agent_phone = b.agent_phone;
if (b.office_phone) o.office_phone = b.office_phone;
+ // Contact/both-sides overlay (TK-10243): listing-agent email (from find-email enrichment) + the
+ // buyer-SIDE agents (from the boughtWith scrape, SOLD listings only). Passed through verbatim so
+ // condos.html can render "both sides" + a mailto without a schema migration.
+ if (b.agent_email) o.agent_email = b.agent_email;
+ if (Array.isArray(b.buyer_agents) && b.buyer_agents.length) o.buyer_agents = b.buyer_agents;
+ if (b.buyer_agent_name) { o.buyer_agent_name = b.buyer_agent_name; o.buyer_firm_name = b.buyer_firm_name; o.buyer_agent_dre = b.buyer_agent_dre; o.buyer_agent_phone = b.buyer_agent_phone; }
if (b.dre) {
o.dre_url = b.dre.dre_url || (b.broker_dre ? `https://www2.dre.ca.gov/PublicASP/pplinfo.asp?License_id=${b.broker_dre}` : undefined);
o.broker_dre_status = b.dre.dre_status;
← 115cbb9 condos.html: add broker/agent contact field toggles (listing
·
back to Commercialrealestate
·
auto-save: 2026-08-05T12:11:52 (4 files) — data/condos-redfi f1d23c6 →