usre: on-demand discoverOne returns graceful throttled status, never records false no_url on cooldown

52035aa9262cb409f065ac282e2b73725f6e7680 · 2026-08-06 10:52:59 -0700 · Steve Abrams

Files touched

M src/enrich/broker_website_discovery.ts

Diff

commit 52035aa9262cb409f065ac282e2b73725f6e7680
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Thu Aug 6 10:52:59 2026 -0700

    usre: on-demand discoverOne returns graceful throttled status, never records false no_url on cooldown
---
 src/enrich/broker_website_discovery.ts | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/enrich/broker_website_discovery.ts b/src/enrich/broker_website_discovery.ts
index 3ea7333..2f2a6dd 100644
--- a/src/enrich/broker_website_discovery.ts
+++ b/src/enrich/broker_website_discovery.ts
@@ -163,23 +163,31 @@ interface BrokerRow { id: number; name: string; firm: string | null; city: strin
 // Discover one agent's site right now (the $0 on-demand path the render calls when
 // a specific agent record is opened — 1 query, never hits the batch throttle wall).
 // Exported so the server can `import { discoverOne }` for lazy enrichment.
-export async function discoverOne(brokerId: number): Promise<{ website: string | null; confidence: number }> {
+export async function discoverOne(brokerId: number): Promise<{ status: 'found' | 'no_url' | 'throttled' | 'missing'; website: string | null; confidence: number }> {
   const r = await query<BrokerRow>(`
     SELECT b.id, b.name, f.name AS firm, b.city, b.license_state
       FROM broker b LEFT JOIN firm f ON f.id = b.firm_id WHERE b.id = $1`, [brokerId]);
-  if (!r.rows.length) return { website: null, confidence: 0 };
+  if (!r.rows.length) return { status: 'missing', website: null, confidence: 0 };
   const b = r.rows[0];
   const q = `"${b.name}" ${b.firm || ''} ${b.city || ''} ${b.license_state} real estate agent`.replace(/\s+/g, ' ').trim();
-  const { hits } = await searchWeb(q);
+  let hits: Hit[];
+  try {
+    ({ hits } = await searchWeb(q));
+  } catch (e) {
+    // Throttled (both free engines bot-gated) — DON'T record a no_url (it wasn't a
+    // real miss, just a cooldown). Leave website_status NULL so it's retried later.
+    if (e instanceof ThrottleError) return { status: 'throttled', website: null, confidence: 0 };
+    throw e;
+  }
   const credible = hits.filter(h => { const hh = host(h.url); return hh && !blocked(hh); }).slice(0, 8);
   const verdict = credible.length ? await verifyAgentSite(b, credible) : { index: -1, confidence: 0 };
   if (verdict.index >= 0 && verdict.confidence >= MIN_CONFIDENCE) {
     const winner = 'https://' + host(credible[verdict.index].url);
     await query(`UPDATE broker SET website=$2, website_status='found', website_confidence=$3, website_discovered_at=NOW() WHERE id=$1`, [b.id, winner, verdict.confidence]);
-    return { website: winner, confidence: verdict.confidence };
+    return { status: 'found', website: winner, confidence: verdict.confidence };
   }
   await query(`UPDATE broker SET website_status='no_url', website_discovered_at=NOW() WHERE id=$1`, [b.id]);
-  return { website: null, confidence: 0 };
+  return { status: 'no_url', website: null, confidence: 0 };
 }
 
 async function main() {

← cacffea usre: agent(broker) own-site discovery — $0 free-SERP + loca · back to Nationalrealestate · usre render: every agent shows firm tie + phone/firm-site/ag f1b3540 →