← back to Commercialrealestate
fetch-sfr-agents: record ALL license numbers per SFR listing (capture every listingAgent, not just [0]) — persist each as a broker w/ role listing/co-listing; only primary writes flat sfr.broker_name; deduped; tested
5bb6bfb84de903218a2bb3f365d87955da862255 · 2026-08-03 11:38:11 -0700 · Steve Abrams
Files touched
M scripts/fetch-sfr-agents.js
Diff
commit 5bb6bfb84de903218a2bb3f365d87955da862255
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Mon Aug 3 11:38:11 2026 -0700
fetch-sfr-agents: record ALL license numbers per SFR listing (capture every listingAgent, not just [0]) — persist each as a broker w/ role listing/co-listing; only primary writes flat sfr.broker_name; deduped; tested
---
scripts/fetch-sfr-agents.js | 76 ++++++++++++++++++++++++++++-----------------
1 file changed, 48 insertions(+), 28 deletions(-)
diff --git a/scripts/fetch-sfr-agents.js b/scripts/fetch-sfr-agents.js
index f971069..961c371 100644
--- a/scripts/fetch-sfr-agents.js
+++ b/scripts/fetch-sfr-agents.js
@@ -37,23 +37,34 @@ const OUT = path.join(ROOT, 'data', 'sfr-agents.json');
const strip = s => s.replace(/^[)\]}'&\s]*\{\}&&/, '').replace(/^[)\]}'\s]+/, '');
const pn = v => (v && typeof v === 'object' ? v.phoneNumber : v) || null;
-function extractAgent(payloadText) {
- let j; try { j = JSON.parse(strip(payloadText)); } catch { return { parseErr: true }; }
+// Return EVERY listing agent (Steve 2026-08-03: record all license numbers), not just [0] — a
+// listing carries the primary agent AND any co-listing agent, each with its own license. Deduped
+// by license||name. Status flags (parseErr/noPayload/suppressed) preserved for the caller.
+function extractAgents(payloadText) {
+ let j; try { j = JSON.parse(strip(payloadText)); } catch { return { parseErr: true, agents: [] }; }
const mh = (j.payload && j.payload.mainHouseInfo) || (j.mainHouseInfo) || null;
- if (!mh) return { noPayload: true };
- const la = Array.isArray(mh.listingAgents) ? mh.listingAgents[0] : null;
- if (!la) return { suppressed: true };
- const ai = la.agentInfo || {};
- const name = (ai.agentName || '').trim();
- if (!name || ai.isAgentNameBlank) return { suppressed: true };
- return {
- name,
- brokerage: (la.brokerName || '').trim() || null,
- license: (la.license || '').trim() || null,
- phone: pn(la.agentPhoneNumber) || pn(la.brokerPhoneNumber) || null,
- email: (la.agentEmailAddress || la.brokerEmailAddress || '').trim() || null,
- isRedfinAgent: !!ai.isRedfinAgent
- };
+ if (!mh) return { noPayload: true, agents: [] };
+ const list = Array.isArray(mh.listingAgents) ? mh.listingAgents : [];
+ if (!list.length) return { suppressed: true, agents: [] };
+ const agents = [], seen = new Set();
+ for (const la of list) {
+ if (!la) continue;
+ const ai = la.agentInfo || {};
+ const name = (ai.agentName || '').trim();
+ if (!name || ai.isAgentNameBlank) continue;
+ const a = {
+ name,
+ brokerage: (la.brokerName || '').trim() || null,
+ license: (la.license || '').trim() || null,
+ phone: pn(la.agentPhoneNumber) || pn(la.brokerPhoneNumber) || null,
+ email: (la.agentEmailAddress || la.brokerEmailAddress || '').trim() || null,
+ isRedfinAgent: !!ai.isRedfinAgent
+ };
+ const k = a.license || a.name;
+ if (!seen.has(k)) { seen.add(k); agents.push(a); }
+ }
+ if (!agents.length) return { suppressed: true, agents: [] };
+ return { agents };
}
async function loadTargets() {
@@ -69,7 +80,7 @@ async function loadTargets() {
return r.rows.filter(x => x.pid && x.pid !== x.source);
}
-async function persistAgent(sfr, a, sourceUrl) {
+async function persistAgent(sfr, a, sourceUrl, role = 'listing') {
const firmId = a.brokerage ? await brokerdb.upsertFirm(a.brokerage) : null;
const r = await brokerdb.pool.query(
`INSERT INTO broker(name, firm_id, phone, email, source, agent_type, license)
@@ -84,12 +95,15 @@ async function persistAgent(sfr, a, sourceUrl) {
const brokerId = r.rows[0].id;
await brokerdb.pool.query(
- `INSERT INTO broker_sfr(broker_id, sfr_id, role) VALUES($1,$2,'listing')
- ON CONFLICT DO NOTHING`, [brokerId, sfr.id]);
+ `INSERT INTO broker_sfr(broker_id, sfr_id, role) VALUES($1,$2,$3)
+ ON CONFLICT DO NOTHING`, [brokerId, sfr.id, role]);
- await brokerdb.pool.query(
- `UPDATE sfr SET broker_name=$2, firm_name=$3, firm_id=$4 WHERE id=$1`,
- [sfr.id, a.name, a.brokerage, firmId]);
+ // Only the PRIMARY listing agent writes the flat sfr.broker_name (a co-agent must not clobber it).
+ if (role === 'listing') {
+ await brokerdb.pool.query(
+ `UPDATE sfr SET broker_name=$2, firm_name=$3, firm_id=$4 WHERE id=$1`,
+ [sfr.id, a.name, a.brokerage, firmId]);
+ }
const prov = [];
if (a.phone) prov.push(['phone', a.phone]);
@@ -143,19 +157,25 @@ async function persistAgent(sfr, a, sourceUrl) {
return { status: r.status, text: await r.text() };
}, u);
if (res.status !== 200 || !res.text) { summary.errors++; results.push({ sfr: t.id, pid: t.pid, status: res.status, error: 'non-200' }); continue; }
- const a = extractAgent(res.text);
- if (a.suppressed || a.noPayload || a.parseErr) {
+ const ex = extractAgents(res.text);
+ if (ex.suppressed || ex.noPayload || ex.parseErr || !ex.agents.length) {
summary.suppressed++;
results.push({ sfr: t.id, pid: t.pid, agent: null, label: 'no public agent (Redfin-listed / suppressed)' });
continue;
}
- await persistAgent(t, a, t.source);
+ // Persist EVERY agent (record all licenses): primary = 'listing', the rest = 'co-listing'.
+ for (let i = 0; i < ex.agents.length; i++) {
+ await persistAgent(t, ex.agents[i], t.source, i === 0 ? 'listing' : 'co-listing');
+ }
+ const a = ex.agents[0]; // primary drives the flat summary/log below
+ const coAgents = ex.agents.length - 1;
summary.captured++;
+ summary.coAgents = (summary.coAgents || 0) + coAgents;
if (a.phone) summary.withPhone++;
if (a.email) summary.withEmail++;
- if (a.license) summary.withLicense++;
- results.push({ sfr: t.id, pid: t.pid, agent: a.name, brokerage: a.brokerage, phone: !!a.phone, email: !!a.email });
- process.stderr.write(` ${t.address}, ${t.city}: ${a.name} / ${a.brokerage || '?'}${a.phone ? ' ☎' : ''}${a.email ? ' ✉' : ''}\n`);
+ if (ex.agents.some(x => x.license)) summary.withLicense++;
+ results.push({ sfr: t.id, pid: t.pid, agent: a.name, brokerage: a.brokerage, phone: !!a.phone, email: !!a.email, agents: ex.agents.length, licenses: ex.agents.filter(x => x.license).map(x => x.license) });
+ process.stderr.write(` ${t.address}, ${t.city}: ${a.name} / ${a.brokerage || '?'}${a.phone ? ' ☎' : ''}${a.email ? ' ✉' : ''}${coAgents ? ` +${coAgents} co-agent` : ''}\n`);
await page.waitForTimeout(500);
} catch (e) { summary.errors++; results.push({ sfr: t.id, pid: t.pid, error: String(e.message).slice(0, 80) }); }
}
← a5ff4d2 condos table DRE# (Cody gate): resolve broker_dre BEFORE the
·
back to Commercialrealestate
·
fetch-redfin-agents (condo): record ALL license numbers per 1e8ea8f →