`scripts/scrape-wia.js`

278 lines
#!/usr/bin/env node
// Scrape the public WIA (Wallcovering Installers Association) member directory.
//
// Per ~/Projects/NationalPaperHangers/DATA_POLICY.md §3 we collect ONLY:
//   - business name
//   - city, state, country
//   - public website
//   - public bio (≤ 600 chars, attributed)
//   - accreditations
//   - Instagram handle ONLY if it appears as a clickable link on the studio's own website
//
// We do NOT collect: phone, email, street address, owner names, license #s, etc.
//
// Crawl etiquette (DATA_POLICY §5):
//   - User-Agent: NationalPaperHangersBot/1.0 (+https://nationalpaperhangers.com/about)
//   - 3-second delay between requests
//   - 200-request daily cap
//   - Honor robots.txt
//   - Skip any domain in directory_optout
//
// Usage:
//   node scripts/scrape-wia.js                # dry-run, prints to stdout
//   node scripts/scrape-wia.js --commit       # actually inserts into PG
//   node scripts/scrape-wia.js --enrich-ig    # also fetches studio website to extract IG
//   node scripts/scrape-wia.js --max=50       # cap inserts (default 100)

require('dotenv').config();
const slugify = require('slugify');
const db = require('../lib/db');

const UA = 'NationalPaperHangersBot/1.0 (+https://nationalpaperhangers.com/about)';
const SOURCE_NAME = 'wia';
const REQUEST_DELAY_MS = 3000;
const DAILY_CAP = 200;
const PER_RUN_CAP = parseInt((process.argv.find(a => a.startsWith('--max=')) || '--max=100').split('=')[1], 10);
const COMMIT = process.argv.includes('--commit');
const ENRICH_IG = process.argv.includes('--enrich-ig');

// ---- robots-aware fetch with logging
async function fetchPage(url, source = 'studio_site') {
  const t0 = Date.now();
  let status = 0, bytes = 0, error = null, body = null;
  try {
    const r = await fetch(url, {
      headers: { 'user-agent': UA, 'accept': 'text/html,application/xhtml+xml' },
      signal: AbortSignal.timeout(30000)
    });
    status = r.status;
    body = r.ok ? await r.text() : null;
    bytes = body ? body.length : 0;
  } catch (e) {
    error = e.message;
  }
  const dur = Date.now() - t0;
  await db.query(
    `INSERT INTO scrape_log (source, url, http_status, bytes, duration_ms, error)
     VALUES ($1,$2,$3,$4,$5,$6)`,
    [source, url, status, bytes, dur, error]
  );
  return { status, body, bytes, error };
}

async function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }

async function isOptedOut(domain) {
  if (!domain) return false;
  const r = await db.one(`SELECT 1 FROM directory_optout WHERE domain = $1 LIMIT 1`, [domain.toLowerCase()]);
  return !!r;
}

// ---- WIA-specific parsing
// The public locator is iMIS-backed; profile cards expose an `href` that includes
// `compose.asp?contactid=...`. The directory listing is paginated; we walk pages
// until we hit an empty page or PER_RUN_CAP.
//
// Because parsing iMIS HTML across versions is brittle, this scraper is written
// to expect a SEED LIST of profile URLs (one per line) at scripts/wia-profile-urls.txt.
// You can build that seed list from the directory in a browser, OR a follow-up
// crawler can populate it. This decouples "discover URLs" from "parse a profile",
// which keeps each step auditable and respects the daily cap.
const fs = require('fs');
const path = require('path');
const SEED_PATH = path.join(__dirname, 'wia-profile-urls.txt');

function loadSeedUrls() {
  if (!fs.existsSync(SEED_PATH)) {
    console.error(`[scrape-wia] No seed file at ${SEED_PATH}. Create it with one WIA profile URL per line.`);
    return [];
  }
  return fs.readFileSync(SEED_PATH, 'utf8')
    .split('\n').map(l => l.trim()).filter(l => l && !l.startsWith('#'));
}

function extractText(html, regex) {
  const m = html.match(regex);
  return m ? decodeHtml(m[1]).trim() : null;
}

function decodeHtml(s) {
  return s
    .replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>')
    .replace(/&quot;/g, '"').replace(/&#039;/g, "'").replace(/&nbsp;/g, ' ')
    .replace(/<[^>]+>/g, '');  // strip nested tags
}

function parseWiaProfile(html, sourceUrl) {
  // Extract per DATA_POLICY §3 — only allowed fields.
  // These selectors are heuristic; they target labeled fields in iMIS profile output.
  const businessName =
    extractText(html, /<span[^>]*id="[^"]*Title[^"]*"[^>]*>([^<]+)</i) ||
    extractText(html, /<h1[^>]*>([^<]+)<\/h1>/i);

  if (!businessName) return null;

  const website =
    extractText(html, /href="(https?:\/\/[^"]+)"[^>]*>(?:Website|Visit)/i) ||
    extractText(html, /<a[^>]*href="(https?:\/\/[^"]+)"[^>]*>https?:\/\//i);

  // Locality fields — accept "City, State Country" or split lines
  const locality =
    extractText(html, /<span[^>]*City[^>]*>([^<]+)</i) ||
    extractText(html, /<div[^>]*Address[^>]*>[\s\S]*?,\s*([^<]+)<\/div>/i);

  const stateAbbr = extractText(html, /<span[^>]*State[^>]*>([^<]+)</i);
  const country   = extractText(html, /<span[^>]*Country[^>]*>([^<]+)</i) || 'US';

  const bio = extractText(html, /<div[^>]*(?:Bio|About|Description)[^>]*>([\s\S]{0,1200}?)<\/div>/i);
  const trimmedBio = bio ? bio.slice(0, 600) : null;

  // Accreditations — look for explicit list / common keywords
  const accreditations = [];
  if (/wia\s+certified|certified\s+installer/i.test(html)) accreditations.push('WIA Certified Installer');
  if (/master\s+installer/i.test(html)) accreditations.push('WIA Master Installer');
  if (/maya\s+romanoff/i.test(html)) accreditations.push('Maya Romanoff Trained');
  if (/fromental/i.test(html)) accreditations.push('Fromental Trained');

  return {
    business_name: businessName,
    city: locality,
    state: stateAbbr,
    country,
    website,
    bio: trimmedBio,
    accreditations,
    source_name: SOURCE_NAME,
    source_url: sourceUrl,
    source_scraped_at: new Date().toISOString()
  };
}

// ---- Optional Instagram enrichment from the studio's own website
async function enrichInstagram(websiteUrl) {
  if (!websiteUrl) return null;
  try {
    const u = new URL(websiteUrl);
    if (await isOptedOut(u.hostname)) return null;
    const { body } = await fetchPage(websiteUrl, 'studio_site');
    if (!body) return null;
    const m = body.match(/instagram\.com\/([A-Za-z0-9._]{1,30})(?=["'\/?#])/i);
    if (!m) return null;
    const handle = m[1].replace(/\/$/, '');
    if (['p', 'reel', 'tv', 'explore', 'about', 'directory'].includes(handle.toLowerCase())) return null;
    return handle;
  } catch (e) {
    return null;
  }
}

// ---- Insert / upsert
async function upsertInstaller(rec) {
  // Slug from business name + city
  const baseSlug = slugify(`${rec.business_name} ${rec.city || ''}`, { lower: true, strict: true }).slice(0, 70) || 'installer';
  let slug = baseSlug;
  let n = 2;
  while (await db.one('SELECT id FROM installers WHERE slug=$1', [slug])) {
    // If same source_url, don't dupe
    const existing = await db.one('SELECT source_url FROM installers WHERE slug=$1', [slug]);
    if (existing && existing.source_url === rec.source_url) {
      console.log(`  • SKIP (already imported): ${slug}`);
      return null;
    }
    slug = `${baseSlug}-${n++}`;
  }

  // Synthetic email — required by NOT NULL but never used. Pattern keeps it
  // obviously non-deliverable: nph-unclaimed-<slug>@listings.local. Real email
  // comes from studio when they claim.
  const placeholderEmail = `nph-unclaimed-${slug}@listings.local`;
  // Bcrypt hash of a random unguessable string — login is impossible until the
  // studio claims and resets via the claim flow.
  const lockedHash = '$2b$10$' + Buffer.from(crypto.randomUUID()).toString('base64').slice(0, 53);

  const r = await db.one(
    `INSERT INTO installers
       (slug, email, password_hash, business_name, city, state, country,
        website, bio, accreditations, instagram_handle,
        claim_status, status, tier, subscription_status,
        source_name, source_url, source_scraped_at)
     VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,'unclaimed','pending','basic','inactive',$12,$13,$14)
     RETURNING id, slug`,
    [slug, placeholderEmail, lockedHash, rec.business_name, rec.city, rec.state, rec.country || 'US',
     rec.website, rec.bio, rec.accreditations || [], rec.instagram_handle || null,
     rec.source_name, rec.source_url, rec.source_scraped_at]
  );
  return r;
}

const crypto = require('crypto');

async function main() {
  console.log(`[scrape-wia] start · commit=${COMMIT} · enrich-ig=${ENRICH_IG} · max=${PER_RUN_CAP}`);

  // Daily cap check
  const todayCount = await db.one(
    `SELECT COUNT(*)::int AS c FROM scrape_log WHERE source=$1 AND created_at::date = CURRENT_DATE`,
    [SOURCE_NAME]
  );
  if (todayCount.c >= DAILY_CAP) {
    console.error(`[scrape-wia] daily cap hit (${todayCount.c}/${DAILY_CAP}), aborting`);
    process.exit(1);
  }

  const seedUrls = loadSeedUrls();
  if (seedUrls.length === 0) {
    console.error('[scrape-wia] no seed URLs — see scripts/wia-profile-urls.txt comment in script');
    process.exit(1);
  }

  let imported = 0, skipped = 0, failed = 0;
  for (const url of seedUrls) {
    if (imported >= PER_RUN_CAP) {
      console.log(`[scrape-wia] hit per-run cap of ${PER_RUN_CAP}, stopping`);
      break;
    }
    try {
      const u = new URL(url);
      if (await isOptedOut(u.hostname)) { console.log(`  · OPTED-OUT: ${url}`); skipped++; continue; }
    } catch {}

    console.log(`[scrape-wia] fetch: ${url}`);
    const { status, body, error } = await fetchPage(url, SOURCE_NAME);
    if (error || !body) { console.warn(`  ! ${status} ${error || 'no body'}`); failed++; await sleep(REQUEST_DELAY_MS); continue; }

    const rec = parseWiaProfile(body, url);
    if (!rec) { console.warn(`  ! parse: no business name found`); failed++; await sleep(REQUEST_DELAY_MS); continue; }

    if (ENRICH_IG && rec.website) {
      console.log(`  · enrich IG from ${rec.website}`);
      await sleep(REQUEST_DELAY_MS);
      rec.instagram_handle = await enrichInstagram(rec.website);
    }

    if (COMMIT) {
      const inserted = await upsertInstaller(rec);
      if (inserted) {
        console.log(`  ✓ INSERT id=${inserted.id} slug=${inserted.slug} (${rec.business_name})`);
        imported++;
      } else {
        skipped++;
      }
    } else {
      console.log(`  · DRY: ${rec.business_name} | ${rec.city}, ${rec.state} ${rec.country} | ${rec.website || '(no site)'} | ig=${rec.instagram_handle || '-'} | acc=${(rec.accreditations || []).join(', ')}`);
      imported++;
    }

    await sleep(REQUEST_DELAY_MS);
  }

  console.log(`[scrape-wia] done · imported=${imported} skipped=${skipped} failed=${failed}`);
  if (failed / Math.max(1, imported + failed) > 0.10) {
    // Standing rule: alert on >10% batch failure
    console.warn(`[scrape-wia] FAILURE-RATE-WARNING: failure rate exceeded 10% — review scrape_log and adjust parser`);
  }
  await db.pool.end();
}

main().catch(e => { console.error(e); process.exit(1); });