li-search: multi-engine rotation + bing/mojeek (built; PAUSED — $0 bulk LinkedIn search is bot-blocked, integrity risk)

298f4ef5784b5da9412d44667e81862b62eaa2cf · 2026-08-12 10:03:01 -0700 · Steve Abrams
Files touched

M scripts/enrich-linkedin-search.js
Diff

commit 298f4ef5784b5da9412d44667e81862b62eaa2cf
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Wed Aug 12 10:03:01 2026 -0700

    li-search: multi-engine rotation + bing/mojeek (built; PAUSED — $0 bulk LinkedIn search is bot-blocked, integrity risk)
---
 scripts/enrich-linkedin-search.js | 82 +++++++++++++++++++++++++--------------
 1 file changed, 52 insertions(+), 30 deletions(-)

diff --git a/scripts/enrich-linkedin-search.js b/scripts/enrich-linkedin-search.js
index 4340441..2b2e3bd 100644
--- a/scripts/enrich-linkedin-search.js
+++ b/scripts/enrich-linkedin-search.js
@@ -1,10 +1,12 @@
-// Multi-source LinkedIn-by-SEARCH ($0 LOCAL) for every RE entity with NO website.
-// ONE polite low-rate DuckDuckGo stream drains each source pool in order (CSLB site-less →
-// rentv_licensed_targets → …). A 2nd concurrent DDG worker would just throttle both, so this
-// is deliberately single-worker + multi-source. Looks up "<name> <city> CA linkedin", takes
-// the first real linkedin.com/(company|in|pub) URL, and accepts it ONLY if a name token
-// (>=4 chars) appears in the slug. NEVER requests linkedin.com. DDG-block => cool down and
-// DO NOT stamp (retry later), so a rate-limit is never mistaken for "no profile".
+// Multi-source, MULTI-ENGINE LinkedIn-by-search ($0 LOCAL) for RE entities with no website.
+// One polite stream drains each source pool in order (CSLB site-less → rentv_licensed_targets
+// → …). For each entity we ROTATE across free search engines (DDG → Bing → Mojeek) so no
+// single engine's rate limit can stall the whole stream — same host-diversity trick that beat
+// the paid API. Query uses `site:linkedin.com`, so we regex linkedin.com/(company|in|pub) URLs
+// straight out of whichever engine responds, and accept one only if a name token (>=4) is in
+// the slug. We NEVER request linkedin.com itself. An engine that 403s/captchas is put on a
+// 5-min cooldown; if ALL engines are cooling we wait, and the row is NOT stamped (retry later)
+// so a rate-limit is never mistaken for "no profile".
 //
 // Usage: node scripts/enrich-linkedin-search.js [--loop] [--batch=N] [--all]
 import { q, pool } from '../src/db.js';
@@ -13,12 +15,14 @@ const flags = new Set(process.argv.slice(2).filter(a => a.startsWith('--') && !a
 const kv = Object.fromEntries(process.argv.slice(2).filter(a => a.includes('=')).map(a => a.slice(2).split('=')));
 const BATCH = Number(kv.batch || 30);
 const LA_ONLY = !flags.has('--all');
-const DELAY = Number(process.env.LI_DELAY_MS || 2200);
+const DELAY = Number(process.env.LI_DELAY_MS || 4000);
 const sleep = ms => new Promise(r => setTimeout(r, ms));
-const LI_RE = /https?:\/\/(?:[a-z]{2,3}\.)?linkedin\.com\/(?:company|in|pub|school)\/[A-Za-z0-9._~%\-]+/i;
+const enc = encodeURIComponent;
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36';
+const LI_RE = /https?:\/\/(?:[a-z]{2,3}\.)?linkedin\.com\/(?:company|in|pub|school)\/[A-Za-z0-9._~%\-]+/ig;
 const BAD_LI = /linkedin\.com\/(?:shareArticle|sharing|cws|feed|company\/setup|sales|learning|jobs|pulse|posts|directory)/i;
+const CAPTCHA = /(unusual traffic|are you a (?:human|robot)|captcha|verify you are)/i;
 
-// each source: how to read a pool of un-enriched rows and where to write the result back
 const SOURCES = [
   { key: 'cslb', table: 'cslb_raw', id: '"LicenseNo"', name: '"BusinessName"', city: '"City"',
     where: `"PrimaryStatus"='CLEAR' AND website IS NULL${LA_ONLY ? ` AND "County"='Los Angeles'` : ''}` },
@@ -26,16 +30,13 @@ const SOURCES = [
     where: `(website IS NULL OR website='')` },
 ];
 
-async function ddgLinks(query) {
-  const res = await fetch('https://html.duckduckgo.com/html/?q=' + encodeURIComponent(query),
-    { headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36' } });
-  if (res.status === 403 || res.status === 429) throw new Error('ddg-blocked');
-  if (!res.ok) throw new Error('ddg ' + res.status);
-  const html = await res.text();
-  const out = []; const re = /result__a"[^>]*href="([^"]+)"/g; let m;
-  while ((m = re.exec(html)) && out.length < 8) { const u = m[1].match(/uddg=([^&]+)/); out.push(u ? decodeURIComponent(u[1]) : m[1]); }
-  return out;
-}
+const ENGINES = [
+  { name: 'ddg',    url: qy => `https://html.duckduckgo.com/html/?q=${enc(qy)}`, cool: 0 },
+  { name: 'bing',   url: qy => `https://www.bing.com/search?q=${enc(qy)}`,       cool: 0 },
+  { name: 'mojeek', url: qy => `https://www.mojeek.com/search?q=${enc(qy)}`,     cool: 0 },
+];
+let rr = 0;                                            // round-robin cursor
+
 function slugMatches(name, url) {
   const slug = url.toLowerCase().replace(/[^a-z0-9]/g, '');
   const toks = String(name || '').toLowerCase()
@@ -43,13 +44,34 @@ function slugMatches(name, url) {
     .replace(/[^a-z0-9 ]/g, ' ').split(/\s+/).filter(t => t.length >= 4);
   return toks.some(t => slug.includes(t));
 }
+function extractLinkedin(html, name) {
+  const hits = [...new Set((html.match(LI_RE) || []).map(u => u.replace(/\/$/, '').replace(/\\/g, '')))]
+    .filter(u => !BAD_LI.test(u));
+  const ok = hits.filter(u => slugMatches(name, u));
+  const pick = ok.find(u => /\/company\//i.test(u)) || ok[0];
+  return pick ? pick.replace(/\?.*$/, '') : null;
+}
+
+// try engines in rotation; return {link} on a responding engine, or throw 'all-cooling'
 async function findLinkedin(name, city) {
-  const links = await ddgLinks(`${name} ${city || ''} CA linkedin`);   // throws 'ddg-blocked' up
-  const li = links.find(u => LI_RE.test(u) && !BAD_LI.test(u));
-  return (li && slugMatches(name, li)) ? li.replace(/\/$/, '').replace(/\?.*$/, '') : null;
+  const query = `${name} ${city || ''} CA site:linkedin.com`;
+  const now = Date.now();
+  for (let k = 0; k < ENGINES.length; k++) {
+    const e = ENGINES[(rr + k) % ENGINES.length];
+    if (e.cool > now) continue;                        // engine on cooldown — skip
+    try {
+      const res = await fetch(e.url(query), { headers: { 'User-Agent': UA, 'Accept-Language': 'en-US' }, signal: AbortSignal.timeout(9000) });
+      if (res.status === 403 || res.status === 429) { e.cool = Date.now() + 300000; continue; }
+      if (!res.ok) { e.cool = Date.now() + 60000; continue; }
+      const html = await res.text();
+      if (CAPTCHA.test(html.slice(0, 4000))) { e.cool = Date.now() + 300000; continue; }
+      rr++;                                            // advance rotation only on a real response
+      return { link: extractLinkedin(html, name), engine: e.name };
+    } catch { e.cool = Date.now() + 60000; continue; }
+  }
+  throw new Error('all-cooling');
 }
 
-// pick a batch from the first non-empty source (drains cslb, then rentv, …)
 async function pick() {
   for (const s of SOURCES) {
     const rows = (await q(`SELECT ${s.id} AS id, ${s.name} AS name, ${s.city} AS city
@@ -67,21 +89,21 @@ async function main() {
     if (!rows.length) { console.log('\n✔ multi-source LinkedIn search complete (all pools drained)'); break; }
     if (src.key !== curKey) { curKey = src.key; console.log(`\n▶ source: ${src.key} (${src.table})`); }
     for (const c of rows) {
-      let link = null;
-      try { link = await findLinkedin(c.name, c.city); }
+      let r = null;
+      try { r = await findLinkedin(c.name, c.city); }
       catch (e) {
-        if (e.message === 'ddg-blocked') { console.log('  ⏸ DDG blocked — cooling 60s (no stamp; retry)'); await sleep(60000); continue; }
-        // other error → treat as attempted, stamp and move on
+        if (e.message === 'all-cooling') { console.log('  ⏸ all engines cooling — wait 90s (no stamp; retry)'); await sleep(90000); continue; }
       }
+      const link = r ? r.link : null;
       await q(`UPDATE ${src.table} SET linkedin=COALESCE($2::text,linkedin),
                  linkedin_source=CASE WHEN $2::text IS NOT NULL THEN 'search' ELSE linkedin_source END,
                  contacts_enriched_at=now() WHERE ${src.id}=$1`, [c.id, link]);
-      done++; if (link) { li++; console.log(`  [${done}] ${String(c.name).slice(0, 34).padEnd(34)} → ${link}`); }
+      done++; if (link) { li++; console.log(`  [${done}] ${String(c.name).slice(0, 32).padEnd(32)} → ${link}  (${r.engine})`); }
       await sleep(DELAY);
     }
     console.log(`— ${curKey}: ${done} processed · ${li} linkedin — $0`);
     if (!flags.has('--loop')) break;
   }
-  console.log(`\nTotal: ${done} processed, ${li} linkedin. $0 (search, local).`);
+  console.log(`\nTotal: ${done} processed, ${li} linkedin. $0 (multi-engine search, local).`);
 }
 main().catch(e => { console.error('li-search error:', e.message); process.exitCode = 1; }).finally(() => pool.end());
← cefbc8e contacts: per-row try/catch so one bad URL can't crash a sha · back to La Socrata Ingester · yoloforever c1: disk-space preflight in run-refresh.sh (skip da8cbba →