enrich: domain-guessing pipeline (scripts/enrich-domain.js) — the real max-it unlock

012231aa2c259101970ff3328992d535e652b9dd · 2026-08-12 08:20:14 -0700 · steve

DDG rate-limits by IP so parallel search self-blocks. Domain-guessing hits each
contractor's OWN candidate domains (different hosts = no shared rate limit) + local
model verifies the page belongs to them. Test: 5/10 real sites, $0, fully parallel.

Files touched

A scripts/enrich-domain.js

Diff

commit 012231aa2c259101970ff3328992d535e652b9dd
Author: steve <steve@designerwallcoverings.com>
Date:   Wed Aug 12 08:20:14 2026 -0700

    enrich: domain-guessing pipeline (scripts/enrich-domain.js) — the real max-it unlock
    
    DDG rate-limits by IP so parallel search self-blocks. Domain-guessing hits each
    contractor's OWN candidate domains (different hosts = no shared rate limit) + local
    model verifies the page belongs to them. Test: 5/10 real sites, $0, fully parallel.
---
 scripts/enrich-domain.js | 101 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 101 insertions(+)

diff --git a/scripts/enrich-domain.js b/scripts/enrich-domain.js
new file mode 100644
index 0000000..e1bb7d8
--- /dev/null
+++ b/scripts/enrich-domain.js
@@ -0,0 +1,101 @@
+// Contractor enrichment via DOMAIN-GUESSING (READ-ONLY writes to cslb_raw, $0 local).
+// The right max-it architecture: instead of one rate-limited search engine, guess each
+// contractor's OWN candidate domains and HTTP-check them — every request hits a DIFFERENT
+// host, so there's no shared rate limit and it parallelizes freely. A local Ollama model
+// verifies the fetched page actually belongs to the contractor + extracts email.
+//
+// Usage: node scripts/enrich-domain.js [--loop] [--batch=N] [--shard=i/N] [--ollama=URL] [--model=M] [--all]
+import { q, pool } from '../src/db.js';
+
+const flags = new Set(process.argv.slice(2).filter(a => a.startsWith('--') && !a.includes('=')));
+const kv = Object.fromEntries(process.argv.slice(2).filter(a => a.includes('=')).map(a => a.slice(2).split('=')));
+const OLLAMA = kv.ollama || 'http://127.0.0.1:11434';
+const MODEL = kv.model || 'qwen2.5:latest';
+const BATCH = Number(kv.batch || 50);
+const LA_ONLY = !flags.has('--all');
+const [SHARD_I, SHARD_N] = (kv.shard || '0/1').split('/').map(Number);
+const SHARD = SHARD_N > 1 ? `AND (abs(hashtext(c."LicenseNo")) % ${SHARD_N}) = ${SHARD_I}` : '';
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+// business name -> candidate domains (their own hosts — no shared rate limit)
+function candidates(name) {
+  const base = String(name).toLowerCase().replace(/&/g, 'and').replace(/[^a-z0-9 ]/g, ' ').replace(/\s+/g, ' ').trim();
+  const noSuffix = base.replace(/\b(inc|llc|corp|corporation|company|co|ltd|lp|the|construction|development|builders|building|group|enterprises|services)\b/g, ' ').replace(/\s+/g, ' ').trim();
+  const compact = s => s.replace(/\s+/g, '');
+  const words = base.split(' ').filter(Boolean);
+  const set = new Set();
+  set.add(compact(base));                        // fullnameconstruction
+  if (noSuffix && noSuffix !== base) set.add(compact(noSuffix)); // fullname
+  if (words.length >= 2) set.add(compact(words.slice(0, 2).join(''))); // firsttwo
+  if (words.length >= 2) set.add(words.map(w => w[0]).join(''));       // initials
+  const doms = [];
+  for (const s of set) if (s.length >= 3 && s.length <= 40) for (const tld of ['.com', '.net']) doms.push(s + tld);
+  return [...new Set(doms)].slice(0, 8);
+}
+
+async function tryDomain(dom) {
+  try {
+    const res = await fetch('https://' + dom, { headers: { 'User-Agent': 'Mozilla/5.0' }, signal: AbortSignal.timeout(6000), redirect: 'follow' });
+    if (!res.ok) return null;
+    const ct = res.headers.get('content-type') || '';
+    if (!/html/.test(ct)) return null;
+    const html = (await res.text()).slice(0, 40000);
+    const text = html.replace(/<script[\s\S]*?<\/script>/gi, ' ').replace(/<style[\s\S]*?<\/style>/gi, ' ').replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 1400);
+    const email = (html.match(/[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}/i) || [])[0] || null;
+    return { url: 'https://' + dom, text, email };
+  } catch { return null; }
+}
+
+async function verify(c, hits) {
+  const prompt = `Which of these websites (if any) is the OFFICIAL site of this contractor? Only match if the page clearly is this business (name and, ideally, city/phone appear).
+Contractor: ${c.BusinessName}, ${c.City} CA ${c.ZIPCode}, phone ${c.BusinessPhone || 'n/a'}.
+${hits.map((h, i) => `${i + 1}. ${h.url}\n   ${h.text.slice(0, 600)}`).join('\n')}
+Return ONLY strict JSON: {"index": number|null (1-based, or null if none match), "confidence": "high"|"low"|"none"}`;
+  try {
+    const res = await fetch(OLLAMA + '/api/generate', { method: 'POST', headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ model: MODEL, prompt, stream: false, format: 'json', options: { temperature: 0 } }) });
+    const d = await res.json();
+    const j = JSON.parse(d.response);
+    const pick = j.index && hits[j.index - 1];
+    return pick ? { website: pick.url, email: pick.email, confidence: j.confidence || 'low' } : { website: null, email: null, confidence: 'none' };
+  } catch { return { website: null, email: null, confidence: 'none' }; }
+}
+
+async function pick() {
+  const laFilter = LA_ONLY ? `AND "County"='Los Angeles'` : '';
+  return (await q(`
+    SELECT c."LicenseNo", c."BusinessName", c."City", c."ZIPCode", c."BusinessPhone"
+    FROM cslb_raw c
+    WHERE c."PrimaryStatus"='CLEAR' AND c."BusinessName" IS NOT NULL AND c.enriched_at IS NULL ${laFilter} ${SHARD}
+    ORDER BY c."Classifications(s)" LIKE 'B%' DESC LIMIT ${BATCH}`)).rows;
+}
+
+async function enrichOne(c) {
+  let fields = { website: null, email: null, linkedin: null, confidence: 'none' };
+  try {
+    const doms = candidates(c.BusinessName);
+    const hits = (await Promise.all(doms.map(tryDomain))).filter(Boolean);
+    if (hits.length) { const v = await verify(c, hits); if (v.website) fields = { ...fields, ...v }; }
+  } catch (e) { fields.confidence = 'error:' + e.message.slice(0, 30); }
+  await q(`UPDATE cslb_raw SET website=$2, email=COALESCE($3,email), enrich_confidence=$4, enrich_source='domain-guess+local-llm', enriched_at=now() WHERE "LicenseNo"=$1`,
+    [c.LicenseNo, fields.website, fields.email, fields.confidence]);
+  return fields;
+}
+
+async function main() {
+  let done = 0, found = 0;
+  for (;;) {
+    const batch = await pick();
+    if (!batch.length) { console.log(`\n✔ complete — no unenriched left (${LA_ONLY ? 'LA County' : 'all'})`); break; }
+    for (const c of batch) {
+      const f = await enrichOne(c);
+      done++; if (f.website) found++;
+      if (f.website) console.log(`  [${done}] ${c.BusinessName.slice(0, 32).padEnd(32)} → ${f.website} (${f.confidence})`);
+    }
+    const rem = Number((await q(`SELECT count(*) c FROM cslb_raw WHERE "PrimaryStatus"='CLEAR' AND enriched_at IS NULL ${LA_ONLY ? `AND "County"='Los Angeles'` : ''}`)).rows[0].c);
+    console.log(`— ${done} processed, ${found} sites, ~${rem.toLocaleString()} left — $0`);
+    if (!flags.has('--loop')) break;
+  }
+  console.log(`\nTotal: ${done} processed, ${found} websites. $0 (domain-guess + local model).`);
+}
+main().catch(e => { console.error('enrich-domain error:', e.message); process.exitCode = 1; }).finally(() => pool.end());

← 14a832a enrich: don't lose DDG-rate-limited rows (mark ddg-blocked, · back to La Socrata Ingester · Property drill-down (scripts/property.js): full permit/trade fa030eb →