← back to Marketing Command Center
clients: browser+local-model contact enrichment (direct Instagram/LinkedIn + email gap-fill)
e0ae6f56aaca9909f4106088b6b6b084b30adba8 · 2026-07-16 11:17:41 -0700 · steve
- enrich-contacts-browser.mjs: headless Chromium (Playwright) renders JS-only
sites the HTTP crawler misses; pulls direct instagram/linkedin/tel from the
DOM + mailto; falls back to local Ollama (qwen2.5) to deobfuscate emails. $0.
- Panel shows DIRECT IG/LinkedIn (gold ↗) when resolved, search (⌕) otherwise;
CSV export carries the direct links. Wired into refresh-clients.sh weekly.
Files touched
M public/panels/clients.jsA scripts/enrich-contacts-browser.mjsM scripts/refresh-clients.sh
Diff
commit e0ae6f56aaca9909f4106088b6b6b084b30adba8
Author: steve <steve@designerwallcoverings.com>
Date: Thu Jul 16 11:17:41 2026 -0700
clients: browser+local-model contact enrichment (direct Instagram/LinkedIn + email gap-fill)
- enrich-contacts-browser.mjs: headless Chromium (Playwright) renders JS-only
sites the HTTP crawler misses; pulls direct instagram/linkedin/tel from the
DOM + mailto; falls back to local Ollama (qwen2.5) to deobfuscate emails. $0.
- Panel shows DIRECT IG/LinkedIn (gold ↗) when resolved, search (⌕) otherwise;
CSV export carries the direct links. Wired into refresh-clients.sh weekly.
---
public/panels/clients.js | 8 +--
scripts/enrich-contacts-browser.mjs | 118 ++++++++++++++++++++++++++++++++++++
scripts/refresh-clients.sh | 8 +++
3 files changed, 130 insertions(+), 4 deletions(-)
diff --git a/public/panels/clients.js b/public/panels/clients.js
index 37755ff..3890113 100644
--- a/public/panels/clients.js
+++ b/public/panels/clients.js
@@ -78,8 +78,8 @@ window.MCC_PANELS['clients'] = {
const cell = v => { v = String(v == null ? '' : v); return /[",\n]/.test(v) ? '"' + v.replace(/"/g, '""') + '"' : v; };
let headers, line;
if (d.kind === 'prospects') {
- headers = ['Business', 'City', 'Phone', 'Website', 'Email', 'LinkedIn Search', 'Instagram Search'];
- line = r => [r._raw.title, r._raw.city, r._raw.phone, r._raw.website, r._raw.email, liURL(r.q), igURL(r.q)];
+ headers = ['Business', 'City', 'Phone', 'Website', 'Email', 'LinkedIn', 'Instagram'];
+ line = r => [r._raw.title, r._raw.city, r._raw.phone, r._raw.website, r._raw.email, r._raw.linkedin || liURL(r.q), r._raw.instagram || igURL(r.q)];
} else {
headers = ['Account', 'Company', 'Name', 'City', 'State', 'Email', 'Phone', 'LinkedIn Search', 'Instagram Search'];
line = r => [r._raw.account, r._raw.company, r._raw.name, r._raw.city, r._raw.state, r._raw.email, r._raw.phone, liURL(r.q), igURL(r.q)];
@@ -109,8 +109,8 @@ window.MCC_PANELS['clients'] = {
${sub}
<span style="display:flex;gap:12px;flex-wrap:wrap;margin-top:2px;">${mail} ${web}</span>
</span>
- <a class="btn ghost" href="${liURL(r.q)}" target="_blank" rel="noopener" style="font-size:10.5px;padding:4px 8px;" onclick="event.stopPropagation()">in ↗</a>
- <a class="btn ghost" href="${igURL(r.q)}" target="_blank" rel="noopener" style="font-size:10.5px;padding:4px 8px;" onclick="event.stopPropagation()">IG ↗</a>
+ <a class="btn ${r._raw.linkedin ? 'gold' : 'ghost'}" href="${esc(r._raw.linkedin || liURL(r.q))}" target="_blank" rel="noopener" title="${r._raw.linkedin ? 'direct LinkedIn' : 'LinkedIn search'}" style="font-size:10.5px;padding:4px 8px;" onclick="event.stopPropagation()">in ${r._raw.linkedin ? '↗' : '⌕'}</a>
+ <a class="btn ${r._raw.instagram ? 'gold' : 'ghost'}" href="${esc(r._raw.instagram || igURL(r.q))}" target="_blank" rel="noopener" title="${r._raw.instagram ? 'direct Instagram' : 'Instagram search'}" style="font-size:10.5px;padding:4px 8px;" onclick="event.stopPropagation()">IG ${r._raw.instagram ? '↗' : '⌕'}</a>
</label>`;
}).join('')}</div>${more > 0 ? `<div class="muted" style="font-size:11px;margin-top:8px;">Showing ${shown.length} of ${matches.length.toLocaleString()} — type in the filter to narrow.</div>` : ''}</div>`;
$('#cl-progress').textContent = `${set.size.toLocaleString()} / ${RECS.length.toLocaleString()} contacted`;
diff --git a/scripts/enrich-contacts-browser.mjs b/scripts/enrich-contacts-browser.mjs
new file mode 100644
index 0000000..ef4072f
--- /dev/null
+++ b/scripts/enrich-contacts-browser.mjs
@@ -0,0 +1,118 @@
+#!/usr/bin/env node
+/*
+ * enrich-contacts-browser.mjs — deep contact enrichment with a REAL browser +
+ * a LOCAL model. Opens each business website in headless Chromium (renders JS
+ * that the HTTP crawler can't see), visits a contact/about page, and logs:
+ * • email (DOM mailto first; local Ollama model deobfuscates "name [at] domain" text if none)
+ * • instagram (real profile link from the site — direct, not a search)
+ * • linkedin (company/in profile link)
+ * • phone (tel: link)
+ *
+ * Layers on top of a prospects-<slug>.json (from crawl-csv-emails.py): reads it,
+ * enriches each business in place, writes it back. Local + free ($0). Nothing
+ * is uploaded to Constant Contact.
+ *
+ * node scripts/enrich-contacts-browser.mjs public/data/prospects-sf-architects.json [--limit N] [--workers 6] [--model qwen2.5:latest]
+ */
+import { readFileSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+
+// Reuse an existing Playwright install (browsers already cached in ms-playwright).
+const require = createRequire(import.meta.url);
+let chromium;
+for (const p of ['/Users/macstudio3/Projects/Designer-Wallcoverings/node_modules/playwright',
+ '/Users/macstudio3/Projects/astek-landing/node_modules/playwright',
+ 'playwright']) {
+ try { ({ chromium } = require(p)); break; } catch {}
+}
+if (!chromium) { console.error('playwright not found'); process.exit(1); }
+
+const args = process.argv.slice(2);
+const FILE = args.find(a => !a.startsWith('--'));
+const opt = (k, d) => { const i = args.indexOf('--' + k); return i >= 0 ? args[i + 1] : d; };
+const LIMIT = Number(opt('limit', 0));
+const WORKERS = Number(opt('workers', 6));
+const MODEL = opt('model', 'qwen2.5:latest');
+const OLLAMA = 'http://127.0.0.1:11434/api/generate';
+
+const data = JSON.parse(readFileSync(FILE, 'utf8'));
+let biz = data.businesses || [];
+if (LIMIT) biz = biz.slice(0, LIMIT);
+
+const EMAIL_RE = /[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}/g;
+const JUNK = /(sentry|wix\.com|wixpress|godaddy|squarespace|example\.|domain\.com|yourdomain|\.png|\.jpg|\.svg|sentry\.io|@2x)/i;
+const norm = u => { try { return new URL(u.startsWith('http') ? u : 'http://' + u).href; } catch { return null; } };
+const host = u => { try { return new URL(u).hostname.replace(/^www\./, ''); } catch { return ''; } };
+
+async function ollamaEmail(text, siteHost) {
+ try {
+ const r = await fetch(OLLAMA, { method: 'POST', headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ model: MODEL, think: false, stream: false,
+ prompt: `From this business website contact text, extract the single best PUBLIC contact email address (prefer one on the @${siteHost} domain, or info@/hello@/contact@). If a real email is present, reply with ONLY that email. If none, reply with exactly NONE.\n\n---\n${text.slice(0, 3500)}\n---` }) });
+ const j = await r.json();
+ const m = (j.response || '').match(EMAIL_RE);
+ return m && !JUNK.test(m[0]) ? m[0].toLowerCase() : '';
+ } catch { return ''; }
+}
+
+async function grabLinks(page) {
+ return page.$$eval('a[href]', as => as.map(a => a.getAttribute('href') || '').filter(Boolean)).catch(() => []);
+}
+function pickSocial(hrefs, re) {
+ for (const h of hrefs) { const m = h.match(re); if (m) return m[0].split('?')[0].replace(/\/$/, ''); }
+ return '';
+}
+
+async function enrichOne(ctx, b) {
+ const url = norm(b.website); if (!url) return;
+ const siteHost = host(url);
+ const page = await ctx.newPage();
+ try {
+ await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 15000 });
+ let hrefs = await grabLinks(page);
+ // hop to a contact/about page if present
+ const contact = hrefs.find(h => /contact|about|connect|team/i.test(h));
+ let text = await page.evaluate(() => document.body ? document.body.innerText : '').catch(() => '');
+ if (contact) {
+ try { await page.goto(new URL(contact, url).href, { waitUntil: 'domcontentloaded', timeout: 12000 });
+ hrefs = hrefs.concat(await grabLinks(page));
+ text += '\n' + await page.evaluate(() => document.body ? document.body.innerText : '').catch(() => ''); } catch {}
+ }
+ // deterministic first
+ const ig = pickSocial(hrefs, /https?:\/\/(?:www\.)?instagram\.com\/(?!p\/|explore|accounts)[A-Za-z0-9_.]+/i);
+ const li = pickSocial(hrefs, /https?:\/\/(?:www\.)?linkedin\.com\/(?:company|in)\/[A-Za-z0-9_%\-]+/i);
+ const tel = pickSocial(hrefs, /tel:\+?[0-9().\-\s]{7,}/i);
+ let email = '';
+ const mailto = hrefs.find(h => /^mailto:/i.test(h));
+ if (mailto) { const e = mailto.replace(/^mailto:/i, '').split('?')[0].trim(); if (EMAIL_RE.test(e) && !JUNK.test(e)) email = e.toLowerCase(); }
+ if (!email) { const m = (text.match(EMAIL_RE) || []).filter(e => !JUNK.test(e)); if (m.length) email = m[0].toLowerCase(); }
+ if (!email) email = await ollamaEmail(text, siteHost); // local-model fallback for obfuscated emails
+ if (ig) b.instagram = ig;
+ if (li) b.linkedin = li;
+ if (tel) b.phone = b.phone || tel.replace(/^tel:/i, '').trim();
+ if (email && !b.email) b.email = email;
+ if (email) b.emails = Array.from(new Set([email, ...(b.emails || [])]));
+ } catch { /* unreachable site */ } finally { await page.close().catch(() => {}); }
+}
+
+const browser = await chromium.launch({ headless: true });
+const ctx = await browser.newContext({ userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36' });
+let done = 0, ig = 0, li = 0, mail = 0;
+const queue = biz.slice();
+async function worker() {
+ while (queue.length) {
+ const b = queue.shift();
+ await enrichOne(ctx, b);
+ done++; if (b.instagram) ig++; if (b.linkedin) li++; if (b.email) mail++;
+ if (done % 10 === 0 || done === biz.length) process.stdout.write(`\r ${done}/${biz.length} · ${mail} email · ${ig} instagram · ${li} linkedin `);
+ }
+}
+await Promise.all(Array.from({ length: WORKERS }, worker));
+await browser.close();
+
+data.with_email = (data.businesses || []).filter(b => b.email).length;
+data.with_instagram = (data.businesses || []).filter(b => b.instagram).length;
+data.with_linkedin = (data.businesses || []).filter(b => b.linkedin).length;
+data.enriched = new Date().toISOString().slice(0, 10);
+writeFileSync(FILE, JSON.stringify(data));
+console.log(`\n✓ ${FILE} — ${data.with_email} email · ${data.with_instagram} instagram · ${data.with_linkedin} linkedin [browser + ${MODEL}, $0 local]`);
diff --git a/scripts/refresh-clients.sh b/scripts/refresh-clients.sh
index 5013a9b..1e41a38 100755
--- a/scripts/refresh-clients.sh
+++ b/scripts/refresh-clients.sh
@@ -36,6 +36,14 @@ crawl "$SRC/*nyc*dataset_crawler-google-places*" nyc-arc
crawl "$SRC/*Santa_Barbara*dataset_crawler-google-places*" santa-barbara
crawl "$SRC/dataset_crawler-google-places_2026-07-10_07-01-32-993* $SRC/*_07-01-32-993*" places-batch-1
+# 2b. Deep browser + local-model enrichment — direct Instagram/LinkedIn links +
+# email gap-fill on JS-rendered sites the HTTP pass can't read. Local ($0).
+echo "-- browser + local-model enrichment"
+shopt -s nullglob
+for pj in public/data/prospects-*.json; do
+ node scripts/enrich-contacts-browser.mjs "$pj" --workers 6 --model qwen2.5:latest || echo "!! enrich failed: $pj"
+done
+
# 3. Manifest the panel reads
echo "-- manifest"
node scripts/build-clients-manifest.js
← ad37a60 refresh-clients.sh: read CSVs from project data/sources (lau
·
back to Marketing Command Center
·
clients panel: search across ALL fields + sort dropdown (nam 1a67441 →