auto-save: 2026-07-21T14:38:01 (1 files) — src/enrich/firm_website_discovery.ts

bf3451dd9659ae36a0e03b2446fdd64b333d5239 · 2026-07-21 14:38:02 -0700 · Steve Abrams
Files touched

M src/enrich/firm_website_discovery.ts
Diff

commit bf3451dd9659ae36a0e03b2446fdd64b333d5239
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Tue Jul 21 14:38:02 2026 -0700

    auto-save: 2026-07-21T14:38:01 (1 files) — src/enrich/firm_website_discovery.ts
---
 src/enrich/firm_website_discovery.ts | 47 +++++++++++++++++++++++++++++++++---
 1 file changed, 43 insertions(+), 4 deletions(-)

diff --git a/src/enrich/firm_website_discovery.ts b/src/enrich/firm_website_discovery.ts
index 75fcb2e..d33d645 100644
--- a/src/enrich/firm_website_discovery.ts
+++ b/src/enrich/firm_website_discovery.ts
@@ -15,6 +15,7 @@
  * timeout, custom UA, escalating backoff + abort on persistent throttle. $0.
  */
 import 'dotenv/config';
+import { chromium, type Browser, type Page } from 'playwright';
 import { pool, query } from '../../db/pool.ts';
 
 const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 usrealestate/0.1 (firm-website-discovery; +local)';
@@ -95,6 +96,25 @@ async function ddgSearch(q: string): Promise<string[]> {
   return out.slice(0, 10);
 }
 
+// Browser-mode search: DDG's JS site via real Chromium. The plain-fetch html
+// endpoint anomaly-flags this IP quickly (HTTP 202 challenge); a real browser
+// passes. One shared page, new search per firm.
+async function ddgBrowserSearch(page: Page, q: string): Promise<string[]> {
+  await page.goto('https://duckduckgo.com/?q=' + encodeURIComponent(q) + '&ia=web', {
+    waitUntil: 'domcontentloaded', timeout: 20_000,
+  });
+  try {
+    await page.waitForSelector('[data-testid="result-title-a"], article a[href^="http"]', { timeout: 8000 });
+  } catch {
+    const body = (await page.textContent('body')) || '';
+    if (/unfortunately|bots|anomaly|browser check/i.test(body)) throw new ThrottleError('ddg browser challenge');
+    return []; // genuinely no results
+  }
+  const hrefs: string[] = await page.$$eval('[data-testid="result-title-a"]',
+    els => els.map(e => (e as HTMLAnchorElement).href).filter(h => /^https?:\/\//.test(h)));
+  return hrefs.slice(0, 10);
+}
+
 function pickWinner(urls: string[], firmName: string): string | null {
   const credible = urls.filter(u => { const h = host(u); return h && !blocked(h); });
   if (!credible.length) return null;
@@ -128,22 +148,40 @@ async function main() {
   console.log(`[discover] run #${runId} · ${queue.length} firms · jittered 2.5-4s/query`);
 
   let ok = 0, miss = 0, err = 0, consecThrottle = 0;
+  let engine: 'fetch' | 'browser' = process.argv.includes('--engine=browser') ? 'browser' : 'fetch';
+  let browser: Browser | null = null;
+  let page: Page | null = null;
+  const ensureBrowser = async () => {
+    if (!page) {
+      browser = await chromium.launch({ headless: true });
+      page = await (await browser.newContext({ userAgent: UA.replace(/ usrealestate.*$/, ''), viewport: { width: 1280, height: 900 } })).newPage();
+    }
+    return page;
+  };
+
   for (let i = 0; i < queue.length; i++) {
     const f = queue[i];
     const q = `${f.name} ${f.hq_city || ''} ${f.license_state} real estate`.replace(/\s+/g, ' ').trim();
     let urls: string[] | null = null;
-    // Up to 3 attempts with escalating backoff on throttle; a firm is only
-    // recorded as a miss when DDG actually returned a results page.
+    // Up to 3 attempts; a plain-fetch throttle escalates to browser mode first,
+    // then to timed backoff. A firm is only recorded as a miss when DDG actually
+    // returned a results page.
     for (let attempt = 0; attempt < 3 && urls === null; attempt++) {
       try {
-        urls = await ddgSearch(q);
+        urls = engine === 'fetch' ? await ddgSearch(q) : await ddgBrowserSearch(await ensureBrowser(), q);
         consecThrottle = 0;
       } catch (e: any) {
         if (e instanceof ThrottleError || /ddg (403|429)/.test(e.message || '')) {
+          if (engine === 'fetch') {
+            console.log(`  [throttle] ${e.message} — switching to browser engine`);
+            engine = 'browser';
+            continue;
+          }
           const wait = [30_000, 90_000, 180_000][attempt];
           console.log(`  [throttle] ${e.message} — backing off ${wait / 1000}s (attempt ${attempt + 1}/3)`);
           await new Promise(res => setTimeout(res, wait));
         } else {
+          console.log(`  [err] ${f.name.slice(0, 40)}: ${(e.message || e).slice(0, 80)}`);
           break; // transport error — skip this firm, don't record
         }
       }
@@ -184,7 +222,8 @@ async function main() {
             notes = notes || $4 WHERE id = $1`,
     [runId, ok, miss, ` · ${ok} found, ${miss} no-result, ${err} search-errors`],
   );
-  console.log(`[discover] done · ${ok} found · ${miss} no-result · ${err} errors`);
+  console.log(`[discover] done · ${ok} found · ${miss} no-result · ${err} errors · engine=${engine}`);
+  if (browser) await (browser as Browser).close().catch(() => {});
   await pool.end();
 }
← 12bd5ee CBSA crosswalk backfill: 1915 counties get cbsa_code from Ce · back to Nationalrealestate · Map page: Top Markets strip (top-8 ranked, clickable to deta dd7b002 →