← back to Commercialrealestate
enrich-condo-brokers (Cody gate): FIX partial-hit clobber (merge prior agents[] so a fallback-only re-scrape can't nuke recorded co-agent licenses) + robust nesting-safe extractAgents (balanced-bracket JSON.parse, regex fallback) — tested 1-level/2-level/malformed
d3c49a38f840bd34b0c0754c77bec256c35d4e47 · 2026-08-03 11:11:04 -0700 · Steve Abrams
Files touched
M scripts/enrich-condo-brokers.js
Diff
commit d3c49a38f840bd34b0c0754c77bec256c35d4e47
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Aug 3 11:11:04 2026 -0700
enrich-condo-brokers (Cody gate): FIX partial-hit clobber (merge prior agents[] so a fallback-only re-scrape can't nuke recorded co-agent licenses) + robust nesting-safe extractAgents (balanced-bracket JSON.parse, regex fallback) — tested 1-level/2-level/malformed
---
scripts/enrich-condo-brokers.js | 80 +++++++++++++++++++++++++++++------------
1 file changed, 57 insertions(+), 23 deletions(-)
diff --git a/scripts/enrich-condo-brokers.js b/scripts/enrich-condo-brokers.js
index 9136fc2..0cd183c 100644
--- a/scripts/enrich-condo-brokers.js
+++ b/scripts/enrich-condo-brokers.js
@@ -49,20 +49,36 @@ const jitter = () => DELAY + Math.floor(Math.random() * 900);
const jload = (p, d) => { try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return d; } };
// ---- Redfin listing-agent extraction (embedded JSON in the page HTML) ----
-function extractBroker(raw) {
- const out = {};
- // Redfin embeds the listing state as an ESCAPED JSON string (\"listingAgents\":[...]) inside the
- // page, so normalize \" -> " before matching. Harmless for our field extraction.
- const html = String(raw).replace(/\\"/g, '"');
- // Record EVERY license number on the listing (Steve 2026-08-03): the listingAgents array carries
- // the primary listing agent AND any co-listing agent, each with its own name/brokerName/license.
- // Capture the full array, split into agent objects (one level of nesting = agentInfo), and pull a
- // {name, firm, dre} from each — not just [0]. Primary stays in the flat fields for back-compat.
- const la = html.match(/"listingAgents":\s*\[(.*?)\]/s);
- if (la) {
- const objs = la[1].match(/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/g) || [];
- const agents = [];
- const seen = new Set();
+// Pull every {name, firm, dre} from the (unescaped) listingAgents array. Prefers a real JSON.parse
+// of the balanced-bracket array (nesting-safe), falls back to a per-object regex, dedups by dre||name.
+function extractAgents(html) {
+ const key = '"listingAgents":';
+ const ki = html.indexOf(key);
+ if (ki < 0) return [];
+ const start = html.indexOf('[', ki);
+ if (start < 0) return [];
+ let depth = 0, end = -1;
+ for (let j = start; j < html.length; j++) {
+ const ch = html[j];
+ if (ch === '[') depth++;
+ else if (ch === ']') { if (--depth === 0) { end = j; break; } }
+ }
+ if (end < 0) return [];
+ const arrText = html.slice(start, end + 1);
+ const raw = [];
+ let parsed = null;
+ try { parsed = JSON.parse(arrText); } catch (e) { parsed = null; }
+ if (Array.isArray(parsed)) {
+ for (const a of parsed) {
+ if (!a || typeof a !== 'object') continue;
+ const name = ((a.agentInfo && a.agentInfo.agentName) || a.agentName || '').toString().trim();
+ const firm = (a.brokerName || '').toString().trim();
+ const licRaw = (a.license || a.breNumber || '').toString().trim();
+ const dre = /^\d{6,8}$/.test(licRaw) ? licRaw : '';
+ if (name || dre) raw.push({ name, firm, dre });
+ }
+ } else {
+ const objs = arrText.match(/\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}/g) || [];
for (const blk of objs) {
const an = blk.match(/"agentName":"([^"]{1,80})"/);
const bn = blk.match(/"brokerName":"([^"]{1,120})"/);
@@ -71,16 +87,30 @@ function extractBroker(raw) {
if (an && an[1].trim()) a.name = an[1].trim();
if (bn && bn[1].trim()) a.firm = bn[1].trim();
if (lic) a.dre = lic[1];
- const k = a.dre || a.name;
- if ((a.name || a.dre) && k && !seen.has(k)) { seen.add(k); agents.push(a); }
- }
- if (agents.length) {
- out.agents = agents; // ALL license-bearing agents
- if (agents[0].name) out.broker_name = agents[0].name; // primary (back-compat)
- if (agents[0].firm) out.firm_name = agents[0].firm;
- if (agents[0].dre) out.broker_dre = agents[0].dre;
+ if (a.name || a.dre) raw.push(a);
}
}
+ const seen = new Set(), out = [];
+ for (const a of raw) { const k = a.dre || a.name; if (k && !seen.has(k)) { seen.add(k); out.push(a); } }
+ return out;
+}
+
+function extractBroker(raw) {
+ const out = {};
+ // Redfin embeds the listing state as an ESCAPED JSON string (\"listingAgents\":[...]) inside the
+ // page, so normalize \" -> " before matching. Harmless for our field extraction.
+ const html = String(raw).replace(/\\"/g, '"');
+ // Record EVERY license number on the listing (Steve 2026-08-03): the listingAgents array carries
+ // the primary listing agent AND any co-listing agent, each with its own name/brokerName/license.
+ // Extract the WHOLE array by balanced brackets, then JSON.parse it (handles arbitrary nesting like
+ // a contactInfo sub-object — Cody gate); fall back to a per-object regex only if it won't parse.
+ const agents = extractAgents(html);
+ if (agents.length) {
+ out.agents = agents; // ALL license-bearing agents
+ if (agents[0].name) out.broker_name = agents[0].name; // primary (back-compat)
+ if (agents[0].firm) out.firm_name = agents[0].firm;
+ if (agents[0].dre) out.broker_dre = agents[0].dre;
+ }
// Fallback flat fields near the top of the page state.
if (!out.broker_name) { const m = html.match(/"listingAgentName":"([^"]{1,80})"/); if (m && m[1].trim()) out.broker_name = m[1].trim(); }
const ap = html.match(/"listingAgentNumber":"([0-9()\-\s.]{7,20})"/); if (ap) out.agent_phone = ap[1].trim();
@@ -209,7 +239,11 @@ async function main() {
// just flag the miss if we had nothing before (so --only-missing can retry it later).
const gotData = b.broker_name || b.firm_name || (Array.isArray(b.agents) && b.agents.length);
if (gotData) {
- brokers[c.id] = { id: c.id, address: c.address, city: c.city, ...b, dre: dre || undefined, fetched_at: new Date().toISOString(), http: status };
+ const prior = brokers[c.id] || {};
+ // Merge over the prior record — a PARTIAL hit (e.g. fallback listingAgentName with no agents
+ // array) must NOT drop the co-agent license list a previous full scrape recorded (Cody gate).
+ const mergedAgents = (Array.isArray(b.agents) && b.agents.length) ? b.agents : (prior.agents || undefined);
+ brokers[c.id] = { ...prior, id: c.id, address: c.address, city: c.city, ...b, agents: mergedAgents, dre: dre || prior.dre || undefined, fetched_at: new Date().toISOString(), http: status };
} else if (!brokers[c.id] || !(brokers[c.id].broker_name || brokers[c.id].broker_dre)) {
brokers[c.id] = { id: c.id, address: c.address, city: c.city, fetched_at: new Date().toISOString(), http: status, blocked: true };
}
← 67623f3 condos: rebuild + send date-ordered all-licenses report (msg
·
back to Commercialrealestate
·
enrich-condo-brokers: cost circuit-breaker (Codex dissent) — 8e1bfaa →