← back to NationalPaperHangers
scripts/scrape-wia-browserbase.js
433 lines
#!/usr/bin/env node
// Scrape WIA public installer locator via Browserbase + Playwright.
//
// Authorization: invoked by Steve with "do for me with browserbase" on 2026-05-05.
// Compliance: DATA_POLICY.md v1.0 — collects ONLY business name, city/state/country,
// public website, accreditations from the public locator listing rows. No per-profile
// drilldown, no email, no phone, no street address.
//
// Usage:
// node scripts/scrape-wia-browserbase.js # dry-run probe (saves screenshot + HTML)
// node scripts/scrape-wia-browserbase.js --commit # also inserts to PG
// node scripts/scrape-wia-browserbase.js --pages=3 # max pages to walk (default 1)
// node scripts/scrape-wia-browserbase.js --debug-only # JUST take a screenshot, no extraction (for tuning)
require('dotenv').config();
require('dotenv').config({ path: require('os').homedir() + '/.claude/skills/browserbase/.env', override: false });
const fs = require('fs');
const path = require('path');
const slugify = require('slugify');
const Browserbase = require('@browserbasehq/sdk').default;
const { chromium } = require('playwright-core');
const db = require('../lib/db');
const SOURCE_NAME = 'wia';
const UA = 'NationalPaperHangersBot/1.0 (+https://nationalpaperhangers.com/about)';
const REQUEST_DELAY_MS = 3000;
const DAILY_CAP = 200;
const COMMIT = process.argv.includes('--commit');
const DEBUG_ONLY = process.argv.includes('--debug-only');
const PAGES = parseInt((process.argv.find(a => a.startsWith('--pages=')) || '--pages=1').split('=')[1], 10);
// Locator URL discovered 2026-05-05 — WordPress page that hosts the
// zip + distance search form. The form submits via GET to root and the
// installer results render on the same locator page (likely via JS).
const LOCATOR_URL = 'https://www.wallcoveringinstallers.org/consumers/locate-a-wallcovering-installer/';
// Major-metro zip codes — walk a small set to seed broad geographic coverage.
// Per DATA_POLICY.md §5, 3s delay between requests + 200/day cap apply.
const SEARCH_ZIPS = [
// Tier 1 — top metros (already covered, dedup will skip)
{ zip: '10001', label: 'New York, NY' },
{ zip: '90048', label: 'Los Angeles, CA' },
{ zip: '60601', label: 'Chicago, IL' },
{ zip: '33131', label: 'Miami, FL' },
{ zip: '75201', label: 'Dallas, TX' },
{ zip: '94103', label: 'San Francisco, CA' },
{ zip: '02108', label: 'Boston, MA' },
{ zip: '98101', label: 'Seattle, WA' },
{ zip: '20001', label: 'Washington, DC' },
{ zip: '30303', label: 'Atlanta, GA' },
{ zip: '85004', label: 'Phoenix, AZ' },
{ zip: '80202', label: 'Denver, CO' },
{ zip: '55401', label: 'Minneapolis, MN' },
{ zip: '63101', label: 'St. Louis, MO' },
{ zip: '37203', label: 'Nashville, TN' },
{ zip: '70112', label: 'New Orleans, LA' },
{ zip: '97201', label: 'Portland, OR' },
{ zip: '84101', label: 'Salt Lake City, UT' },
{ zip: '64108', label: 'Kansas City, MO' },
{ zip: '53202', label: 'Milwaukee, WI' },
// Tier 2 — secondary metros that may have local installers not in 100mi of tier-1
{ zip: '15222', label: 'Pittsburgh, PA' },
{ zip: '44113', label: 'Cleveland, OH' },
{ zip: '46204', label: 'Indianapolis, IN' },
{ zip: '38103', label: 'Memphis, TN' },
{ zip: '28202', label: 'Charlotte, NC' },
{ zip: '33602', label: 'Tampa, FL' },
{ zip: '45202', label: 'Cincinnati, OH' },
{ zip: '14202', label: 'Buffalo, NY' },
{ zip: '89101', label: 'Las Vegas, NV' },
{ zip: '40202', label: 'Louisville, KY' },
{ zip: '23219', label: 'Richmond, VA' },
{ zip: '32202', label: 'Jacksonville, FL' },
{ zip: '19102', label: 'Philadelphia, PA' },
{ zip: '21202', label: 'Baltimore, MD' },
{ zip: '78701', label: 'Austin, TX' },
{ zip: '48226', label: 'Detroit, MI' },
{ zip: '78205', label: 'San Antonio, TX' },
{ zip: '76102', label: 'Fort Worth, TX' },
{ zip: '95814', label: 'Sacramento, CA' },
{ zip: '92101', label: 'San Diego, CA' },
// Tier 3 — sparse / rural geographic coverage
{ zip: '99501', label: 'Anchorage, AK' },
{ zip: '96813', label: 'Honolulu, HI' },
{ zip: '04101', label: 'Portland, ME' },
{ zip: '03101', label: 'Manchester, NH' },
{ zip: '05401', label: 'Burlington, VT' },
{ zip: '57104', label: 'Sioux Falls, SD' },
{ zip: '58102', label: 'Fargo, ND' },
{ zip: '59601', label: 'Helena, MT' },
{ zip: '83702', label: 'Boise, ID' },
{ zip: '87501', label: 'Santa Fe, NM' }
];
const SEARCH_DISTANCE = '100'; // miles
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
async function logRequest(url, status, bytes, durMs, error) {
await db.query(
`INSERT INTO scrape_log (source, url, http_status, bytes, duration_ms, error)
VALUES ($1,$2,$3,$4,$5,$6)`,
[SOURCE_NAME, url, status, bytes, durMs, error]
);
}
async function loadOptOutSet() {
const rows = await db.many(`SELECT domain FROM directory_optout`);
return new Set(rows.map(r => (r.domain || '').toLowerCase()));
}
async function checkDailyCap() {
const r = await db.one(
`SELECT COUNT(*)::int AS c FROM scrape_log WHERE source=$1 AND created_at::date = CURRENT_DATE`,
[SOURCE_NAME]
);
return r.c;
}
async function upsertInstaller(rec) {
const baseSlug = slugify(`${rec.business_name} ${rec.city || ''}`, { lower: true, strict: true }).slice(0, 70) || 'installer';
let slug = baseSlug;
let n = 2;
// Dedupe by (business_name + state) within source_name='wia' so a re-run is
// idempotent. Fall back to (business_name) if state is null.
const exact = rec.state
? await db.one(
'SELECT id, slug FROM installers WHERE LOWER(business_name)=LOWER($1) AND state=$2 AND source_name=$3',
[rec.business_name, rec.state, SOURCE_NAME]
)
: await db.one(
'SELECT id, slug FROM installers WHERE LOWER(business_name)=LOWER($1) AND state IS NULL AND source_name=$2',
[rec.business_name, SOURCE_NAME]
);
if (exact) {
return { id: exact.id, slug: exact.slug, status: 'skip_existing' };
}
while (await db.one('SELECT id FROM installers WHERE slug=$1', [slug])) {
slug = `${baseSlug}-${n++}`;
}
const placeholderEmail = `nph-unclaimed-${slug}@listings.local`;
const lockedHash = '$2b$10$' + require('crypto').randomUUID().replace(/-/g, '').slice(0, 53);
const r = await db.one(
`INSERT INTO installers
(slug, email, password_hash, business_name, city, state, country,
website, bio, accreditations, instagram_handle,
claim_status, status, tier, subscription_status,
source_name, source_url, source_scraped_at)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'unclaimed','pending','basic','inactive',$12,$13,$14)
RETURNING id, slug`,
[slug, placeholderEmail, lockedHash, rec.business_name, rec.city, rec.state, rec.country || 'US',
rec.website, rec.bio, rec.accreditations || [], rec.instagram_handle || null,
SOURCE_NAME, rec.source_url, rec.source_scraped_at]
);
return { ...r, status: 'inserted' };
}
async function main() {
console.log(`[wia-bb] start · commit=${COMMIT} · debug-only=${DEBUG_ONLY} · pages=${PAGES}`);
const apiKey = process.env.BROWSERBASE_API_KEY;
const projectId = process.env.BROWSERBASE_PROJECT_ID;
if (!apiKey || !projectId) {
console.error('[wia-bb] missing BROWSERBASE_API_KEY or BROWSERBASE_PROJECT_ID — check ~/.claude/skills/browserbase/.env');
process.exit(2);
}
const usedToday = await checkDailyCap();
console.log(`[wia-bb] scrape budget today: ${usedToday}/${DAILY_CAP}`);
if (usedToday >= DAILY_CAP) {
console.error('[wia-bb] daily cap hit, aborting');
process.exit(1);
}
const debugDir = path.join(__dirname, '..', 'tmp', 'wia-bb-' + Date.now());
fs.mkdirSync(debugDir, { recursive: true });
const bb = new Browserbase({ apiKey });
let trackedEnd = () => {};
let session;
try {
const { trackedSession } = require(path.join(require('os').homedir(), '.claude/skills/browserbase/track-session.js'));
const wrapped = await trackedSession(bb, projectId, { app: 'nph-wia', note: `pages=${PAGES} commit=${COMMIT}` });
session = wrapped.session;
trackedEnd = wrapped.end;
} catch {
session = await bb.sessions.create({ projectId });
}
console.log(`[wia-bb] session: ${session.id} · live: ${session.debuggerUrl || session.connectUrl}`);
const browser = await chromium.connectOverCDP(session.connectUrl);
const ctx = browser.contexts()[0] || await browser.newContext();
await ctx.setExtraHTTPHeaders({ 'user-agent': UA });
const page = ctx.pages()[0] || await ctx.newPage();
await page.setExtraHTTPHeaders({ 'user-agent': UA });
// Honor robots.txt — quick check on root
try {
const robotsResp = await page.goto('https://www.wallcoveringinstallers.org/robots.txt', { waitUntil: 'load', timeout: 30000 });
const robotsTxt = robotsResp ? await robotsResp.text() : '';
await logRequest('https://www.wallcoveringinstallers.org/robots.txt', robotsResp ? robotsResp.status() : 0, (robotsTxt || '').length, 0, null);
fs.writeFileSync(path.join(debugDir, 'robots.txt'), robotsTxt || '');
if (/Disallow:\s*\/\s*$/m.test(robotsTxt)) {
console.error('[wia-bb] robots.txt disallows root crawling — aborting per policy');
await browser.close();
await trackedEnd();
process.exit(1);
}
} catch (e) {
console.warn('[wia-bb] robots.txt fetch failed (continuing):', e.message);
}
await sleep(REQUEST_DELAY_MS);
// Step 1: Visit the locator page once to warm up state
console.log(`[wia-bb] entry locator: ${LOCATOR_URL}`);
{
const t0 = Date.now();
const resp = await page.goto(LOCATOR_URL, { waitUntil: 'domcontentloaded', timeout: 30000 });
const html = await page.content();
await logRequest(LOCATOR_URL, resp ? resp.status() : 0, html.length, Date.now() - t0, null);
fs.writeFileSync(path.join(debugDir, 'locator-entry.html'), html);
await page.screenshot({ path: path.join(debugDir, 'locator-entry.png'), fullPage: true });
}
if (DEBUG_ONLY) {
console.log(`[wia-bb] debug-only mode — saved locator entry to ${debugDir}`);
await browser.close();
await trackedEnd();
return;
}
// Step 2: Drive the search form for each metro zip
const records = [];
const zipsToSearch = SEARCH_ZIPS.slice(0, PAGES > 1 ? PAGES : SEARCH_ZIPS.length);
for (const z of zipsToSearch) {
console.log(`[wia-bb] search zip=${z.zip} (${z.label})`);
await sleep(REQUEST_DELAY_MS);
const t0 = Date.now();
try {
// Re-load locator page for a clean form each round
await page.goto(LOCATOR_URL, { waitUntil: 'domcontentloaded', timeout: 30000 });
// Fill the zip + distance fields. There are two parallel forms (mobile + desktop)
// both with the same field names. Just fill whichever exists.
await page.evaluate((opts) => {
const setVal = (sel, v) => { const el = document.querySelector(sel); if (el) { el.value = v; el.dispatchEvent(new Event('change', { bubbles: true })); } };
setVal('input[name="zip"]', opts.zip);
const dist = document.querySelector('select[name="distance"]');
if (dist) {
// pick the largest option <= our requested distance, or the largest available
const opts2 = Array.from(dist.options).map(o => parseInt((o.value || '').replace(/[^0-9]/g, ''), 10) || 0);
const targetIdx = opts2.indexOf(Math.max(...opts2));
if (targetIdx >= 0) { dist.selectedIndex = targetIdx; dist.dispatchEvent(new Event('change', { bubbles: true })); }
}
}, { zip: z.zip, distance: SEARCH_DISTANCE });
// Submit by clicking the location-search submit button (NOT the name-search one)
// The button next to the zip/distance has aria-label or button text "Search"
// We pick the FIRST .mbl-search-submit which is the location form on this layout.
const submitClicked = await page.evaluate(() => {
// Prefer a button inside the same form as input[name="zip"]
const zipInput = document.querySelector('input[name="zip"]');
if (!zipInput) return false;
const form = zipInput.closest('form') || zipInput.closest('div');
if (form) {
const btn = form.querySelector('input[type="submit"], button[type="submit"]');
if (btn) { btn.click(); return true; }
}
// Fallback
const anyBtn = document.querySelector('.mbl-search-submit, input[value="Search"]');
if (anyBtn) { anyBtn.click(); return true; }
return false;
});
if (!submitClicked) {
console.warn(` ! could not find submit button for zip=${z.zip}`);
continue;
}
// Wait for results — JS-driven, may take a few seconds
try { await page.waitForLoadState('networkidle', { timeout: 15000 }); } catch {}
await sleep(2000);
const html = await page.content();
const finalUrl = page.url();
fs.writeFileSync(path.join(debugDir, `results-zip-${z.zip}.html`), html);
await page.screenshot({ path: path.join(debugDir, `results-zip-${z.zip}.png`), fullPage: true });
await logRequest(finalUrl, 200, html.length, Date.now() - t0, null);
// Extract installer cards using WIA's real structure (.mbl-result divs).
// We deliberately do NOT touch .mbl-phone, .mbl-phone-mobile, .mbl-email
// per DATA_POLICY §3.
const candidates = await page.evaluate(() => {
const out = [];
document.querySelectorAll('.mbl-result').forEach(card => {
const company = card.querySelector('.mbl-company');
const businessName = company ? (company.textContent || '').trim() : null;
if (!businessName || businessName.length < 2) return;
const addrEl = card.querySelector('.mbl-address');
// .mbl-address contains street + <br> + "City, ST ZIP"
// We pull ONLY the city/state line; never the street.
let cityStateLine = null;
if (addrEl) {
const html = addrEl.innerHTML || '';
const parts = html.split(/<br\s*\/?>/i).map(p => p.replace(/<[^>]+>/g, '').trim()).filter(Boolean);
// The city-state line is the last segment containing ", XX"
cityStateLine = parts.reverse().find(p => /,\s*[A-Z]{2}\b/.test(p)) || null;
}
const websiteEl = card.querySelector('.mbl-website a');
const website = websiteEl ? websiteEl.href : null;
// Bio paragraph follows .mbl-contact-info
const bioEl = card.querySelector('.mbl-contact-info + p, p:not([class])');
const bio = bioEl ? (bioEl.textContent || '').trim().slice(0, 600) : null;
// Accreditations from class signals
const cls = card.className || '';
const accreditations = [];
if (cls.includes('mbl-accredited-result')) accreditations.push('WIA Certified Installer');
if (cls.includes('ris-accreditation')) accreditations.push('RIS Accredited');
out.push({
business_name: businessName,
city_state_line: cityStateLine,
website,
bio,
accreditations
});
});
return out;
});
console.log(` · ${candidates.length} member-locator cards`);
for (const c of candidates) {
// Parse city + state from "City, ST ZIP" — explicitly anchor to comma
// so we never grab a street name as city.
let city = null, state = null, country = 'US';
const line = c.city_state_line || '';
const m1 = line.match(/^([A-Za-z][A-Za-z .'-]{1,40}),\s*([A-Z]{2})(?:\s+\d{4,5})?\s*$/);
if (m1) { city = m1[1].trim(); state = m1[2].trim(); }
else {
const m2 = line.match(/^([A-Za-z][A-Za-z .'-]{1,40}),\s*(United Kingdom|Canada|Australia|New Zealand|Germany|France|Italy|Spain|Netherlands)\s*$/i);
if (m2) { city = m2[1].trim(); country = m2[2].trim(); state = null; }
}
// Reject obvious noise (nav menu items that slipped through, etc.)
if (/^(locate|find|membership|about|contact|search|home|members)/i.test(c.business_name)) continue;
records.push({
business_name: c.business_name,
city, state, country,
website: cleanUrl(c.website),
bio: c.bio || null,
accreditations: c.accreditations || [],
instagram_handle: null,
source_url: finalUrl,
source_scraped_at: new Date().toISOString(),
search_zip: z.zip
});
}
} catch (e) {
console.warn(` ! error on zip=${z.zip}: ${e.message}`);
}
}
// Dedupe by business_name across all zip results
const dedupedByName = new Map();
for (const r of records) {
const k = (r.business_name || '').toLowerCase().trim();
if (!k) continue;
if (!dedupedByName.has(k)) dedupedByName.set(k, r);
}
const uniqRecords = Array.from(dedupedByName.values());
console.log(`[wia-bb] total candidates across all zips: ${records.length}, deduped: ${uniqRecords.length}`);
records.length = 0;
records.push(...uniqRecords);
console.log(`[wia-bb] total candidates: ${records.length}`);
let inserted = 0, skipped = 0;
if (COMMIT) {
// Preload opt-out domains once — avoids N+1 query per record.
const optOutSet = await loadOptOutSet();
for (const rec of records) {
// Drop opted-out domains
try {
if (rec.website) {
const host = new URL(rec.website).hostname.replace(/^www\./, '');
if (optOutSet.has(host.toLowerCase())) { skipped++; continue; }
}
} catch {}
const r = await upsertInstaller(rec);
if (r.status === 'inserted') {
console.log(` ✓ ${r.slug} (${rec.business_name} · ${rec.city || '?'}, ${rec.state || rec.country || '?'})`);
inserted++;
} else {
skipped++;
}
}
} else {
for (const rec of records.slice(0, 20)) {
console.log(` · DRY: ${rec.business_name} · ${rec.city || '?'}, ${rec.state || rec.country || '?'} · ${rec.website || '(no site)'}`);
}
if (records.length > 20) console.log(` · DRY: ...and ${records.length - 20} more`);
}
await browser.close();
await trackedEnd();
console.log(`[wia-bb] done · inserted=${inserted} skipped=${skipped} candidates=${records.length}`);
console.log(`[wia-bb] artifacts: ${debugDir}`);
await db.pool.end();
}
function cleanUrl(u) {
if (!u) return null;
try {
const url = new URL(u);
if (!['http:', 'https:'].includes(url.protocol)) return null;
if (url.hostname.includes('wallcoveringinstallers.org')) return null; // skip internal links
return url.toString();
} catch { return null; }
}
main().catch(e => { console.error(e); process.exit(1); });