← back to La Socrata Ingester

enrich: don't lose DDG-rate-limited rows (mark ddg-blocked, 45s cooldown, throw on 403/429)

14a832a1070d571b8aeeed5c91ff7387fc3ab1f1 · 2026-08-12 08:14:57 -0700 · steve

Max-it fan-out over-pressured DuckDuckGo (403s). Bug: errors stamped enriched_at =
rate-limited contractors marked 'no site' (data loss). Now: 403/429 throws ddg-blocked,
worker cools 45s, degraded rows reset for clean retry. Gentler concurrency.

Files touched

• M scripts/enrich-contractor.js

Diff

commit 14a832a1070d571b8aeeed5c91ff7387fc3ab1f1
Author: steve <steve@designerwallcoverings.com>
Date:   Wed Aug 12 08:14:57 2026 -0700

    enrich: don't lose DDG-rate-limited rows (mark ddg-blocked, 45s cooldown, throw on 403/429)
    
    Max-it fan-out over-pressured DuckDuckGo (403s). Bug: errors stamped enriched_at =
    rate-limited contractors marked 'no site' (data loss). Now: 403/429 throws ddg-blocked,
    worker cools 45s, degraded rows reset for clean retry. Gentler concurrency.
---
 scripts/enrich-contractor.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/enrich-contractor.js b/scripts/enrich-contractor.js
index 01abcf0..0f8ef9d 100644
--- a/scripts/enrich-contractor.js
+++ b/scripts/enrich-contractor.js
@@ -26,6 +26,7 @@ const SHARD = SHARD_N > 1 ? `AND (abs(hashtext(c."LicenseNo")) % ${SHARD_N}) = $
 async function ddg(query) {
   const url = 'https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query);
   const res = await fetch(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' } });
+  if (res.status === 403 || res.status === 429) throw new Error('ddg-blocked'); // rate-limited — DO NOT treat as no-site
   if (!res.ok) throw new Error('ddg ' + res.status);
   const html = await res.text();
   const out = [];
@@ -83,7 +84,7 @@ async function enrichOne(c) {
     const results = await ddg(`${c.BusinessName} ${c.City} California contractor official website`);
     if (results.length) fields = await extract(c, results);
     if (fields.website && DIRECTORY.test(fields.website)) { fields.website = null; if (fields.confidence === 'high') fields.confidence = 'none'; }
-  } catch (e) { fields.confidence = 'error:' + e.message.slice(0, 30); }
+  } catch (e) { fields.confidence = e.message === 'ddg-blocked' ? 'ddg-blocked' : 'error:' + e.message.slice(0, 30); }
   await q(`UPDATE cslb_raw SET website=$2, email=$3, linkedin=$4, enrich_confidence=$5, enrich_source='ddg+local-llm', enriched_at=now() WHERE "LicenseNo"=$1`,
     [c.LicenseNo, fields.website || null, fields.email || null, fields.linkedin || null, fields.confidence || 'none']);
   return fields;
@@ -102,7 +103,9 @@ async function main() {
       const f = await enrichOne(c);
       done++; if (f.website) found++;
       if (done % 10 === 0 || f.website) console.log(`  [${done}] ${c.BusinessName.slice(0, 34).padEnd(34)} ${f.website ? '→ ' + f.website + ' (' + f.confidence + ')' : '· ' + f.confidence}`);
-      await sleep(DELAY);
+      // DDG rate-limit backoff: cool down so it recovers instead of hammering
+      if (f.confidence === 'ddg-blocked') { console.log('  ⏸ DDG rate-limited — cooling 45s'); await sleep(45000); }
+      else await sleep(DELAY);
     }
     const rem = Number((await q(`SELECT count(*) c FROM cslb_raw WHERE "PrimaryStatus"='CLEAR' AND enriched_at IS NULL ${LA_ONLY ? `AND "County"='Los Angeles'` : ''}`)).rows[0].c);
     console.log(`— batch done: ${done} processed, ${found} websites found, ~${rem.toLocaleString()} remaining — $0 (local)`);

← daf3111 enrich: --shard=i/N + --ollama/--model for max-it fan-out ac  ·  back to La Socrata Ingester  ·  enrich: domain-guessing pipeline (scripts/enrich-domain.js) 012231a →