← back to Nationalrealestate

residential enrichment: --asset scoping on free SERP discovery + email crawl reads firm_site.url (chains discovery→email); order both by agent_count DESC for max broker-card leverage (TK-10669)

65cab77d645d42286651ea3605d2cc3abab86412 · 2026-08-18 10:38:35 -0700 · Steve Abrams

Files touched

• M src/enrich/firm_email_crawl.ts
• M src/enrich/firm_website_discovery.ts

Diff

commit 65cab77d645d42286651ea3605d2cc3abab86412
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Aug 18 10:38:35 2026 -0700

    residential enrichment: --asset scoping on free SERP discovery + email crawl reads firm_site.url (chains discovery→email); order both by agent_count DESC for max broker-card leverage (TK-10669)
---
 src/enrich/firm_email_crawl.ts       | 14 ++++++++++----
 src/enrich/firm_website_discovery.ts |  4 ++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/src/enrich/firm_email_crawl.ts b/src/enrich/firm_email_crawl.ts
index d434c38..7cdce7d 100644
--- a/src/enrich/firm_email_crawl.ts
+++ b/src/enrich/firm_email_crawl.ts
@@ -107,11 +107,17 @@ async function crawlFirm(f: FirmRow): Promise<{ id: number; emails: string[]; vi
 }
 
 async function main() {
+  // Website comes from EITHER the Places resolve (firm.website, commercial) OR the
+  // free SERP discovery (firm_site.url, residential) — read both so this crawl chains
+  // off whichever pass populated the site.
   const { rows } = await query<FirmRow>(
-    `SELECT id, name, website FROM firm
-      WHERE asset_class = $1 AND NULLIF(website,'') IS NOT NULL
-        AND id NOT IN (SELECT firm_id FROM firm_contacts WHERE kind = 'email')
-      ORDER BY id ${LIMIT ? 'LIMIT ' + LIMIT : ''}`, [ASSET]);
+    `SELECT f.id, f.name, COALESCE(NULLIF(f.website,''), fs.url) AS website
+       FROM firm f
+       LEFT JOIN firm_site fs ON fs.firm_id = f.id AND NULLIF(fs.url,'') IS NOT NULL
+      WHERE f.asset_class = $1
+        AND COALESCE(NULLIF(f.website,''), fs.url) IS NOT NULL
+        AND f.id NOT IN (SELECT firm_id FROM firm_contacts WHERE kind = 'email')
+      ORDER BY f.agent_count DESC NULLS LAST, f.id ${LIMIT ? 'LIMIT ' + LIMIT : ''}`, [ASSET]);
 
   console.log(`[email-crawl] ${ASSET} · ${rows.length} firms with a site & no email yet · concurrency ${CONCURRENCY} · ${LIVE ? 'LIVE (writing)' : 'DRY-RUN'}`);
 
diff --git a/src/enrich/firm_website_discovery.ts b/src/enrich/firm_website_discovery.ts
index c8d5bc3..d53df6e 100644
--- a/src/enrich/firm_website_discovery.ts
+++ b/src/enrich/firm_website_discovery.ts
@@ -138,6 +138,9 @@ interface FirmRow { id: number; name: string; hq_city: string | null; license_st
 async function main() {
   const argLimit = process.argv.find(a => a.startsWith('--limit='));
   const limit = argLimit ? parseInt(argLimit.split('=')[1], 10) : 200;
+  // Optional asset scoping (TK-10669 residential extension). Fixed literal → no injection.
+  const ASSET_RAW = (process.argv.find(a => a.startsWith('--asset='))?.split('=')[1] || '').toLowerCase();
+  const ASSET_COND = ['commercial', 'residential'].includes(ASSET_RAW) ? `AND f.asset_class = '${ASSET_RAW}'` : '';
 
   const run = await query<{ id: number }>(
     `INSERT INTO ingest_runs (source, notes) VALUES ('firm_discovery', $1) RETURNING id`,
@@ -149,6 +152,7 @@ async function main() {
     SELECT f.id, f.name, f.hq_city, f.license_state
       FROM firm f
      WHERE f.agent_count IS NOT NULL
+       ${ASSET_COND}
        AND NOT EXISTS (SELECT 1 FROM firm_site s WHERE s.firm_id = f.id)
      ORDER BY f.agent_count DESC, f.id
      LIMIT $1

← de5e83f firm email crawl: survive stray HTTP/2 session errors (uncau  ·  back to Nationalrealestate  ·  chore: v0.20.0 (session close — DRE firm contact card shippe 2492689 →