`src/scrape/website_meta.js`

209 lines
// Fetch a small-business website and pull:
//   - <title>, meta description, OG/Twitter meta (title/description/image/site_name)
//   - hero image (og:image first, else first <img> with width/height)
//   - color palette (best-effort — extracts inline hex colors + image-derived if palette skill present;
//     here we use a light heuristic: scrape inline `color:` / `background-color:` hex values from the HTML+CSS)
//   - phone (tel: links + first 10-digit pattern)
//   - email (mailto: links)
//
// Local-only fetch w/ undici. No headless browser.

import { request } from 'undici';
import * as cheerio from 'cheerio';
import { promises as dns } from 'node:dns';
import net from 'node:net';

const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 ' +
           '(KHTML, like Gecko) Chrome/124.0 Safari/537.36 small-business-builder/0.1';

// SSRF guard — reject loopback / link-local / RFC1918 / IPv6-equivalent. Without
// this, the scrape endpoint becomes a probe for internal services
// (127.0.0.1, 169.254.169.254 cloud metadata, 10/8, 172.16/12, 192.168/16, ::1, fc00::/7).
function isPrivateAddr(addr) {
  if (!addr) return true;
  if (net.isIPv4(addr)) {
    const [a, b] = addr.split('.').map(Number);
    if (a === 10) return true;
    if (a === 127) return true;
    if (a === 0) return true;
    if (a === 169 && b === 254) return true;
    if (a === 172 && b >= 16 && b <= 31) return true;
    if (a === 192 && b === 168) return true;
    if (a === 100 && b >= 64 && b <= 127) return true;       // CGNAT
    if (a >= 224) return true;                                 // multicast / reserved
    return false;
  }
  if (net.isIPv6(addr)) {
    const lo = addr.toLowerCase();
    if (lo === '::1' || lo === '::') return true;
    if (lo.startsWith('fe80:') || lo.startsWith('fc') || lo.startsWith('fd')) return true;
    if (lo.startsWith('::ffff:')) {
      const v4 = lo.slice(7);
      if (net.isIPv4(v4)) return isPrivateAddr(v4);
    }
    return false;
  }
  return true;
}

async function assertPublicHost(hostname) {
  // If literal IP, check directly. Otherwise resolve all A/AAAA and reject if ANY
  // address is private (defends against rebinding-style multi-record DNS).
  if (net.isIP(hostname)) {
    if (isPrivateAddr(hostname)) throw new Error('host not allowed');
    return;
  }
  let addrs = [];
  try {
    addrs = await dns.lookup(hostname, { all: true, verbatim: true });
  } catch {
    throw new Error('host not resolvable');
  }
  if (!addrs.length) throw new Error('host not resolvable');
  for (const { address } of addrs) {
    if (isPrivateAddr(address)) throw new Error('host not allowed');
  }
}

export async function fetchHtml(url, timeoutMs = 15000) {
  let parsed;
  try { parsed = new URL(url); } catch { throw new Error('invalid url'); }
  if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {
    throw new Error('only http/https supported');
  }
  await assertPublicHost(parsed.hostname);

  // undici v7 dropped maxRedirections on the low-level request(); use global
  // fetch() which transparently follows up to 20 redirects and re-validates the
  // final hostname is public via assertPublicHost only at the entry. Anything
  // sketchier gets rejected.
  const ac = new AbortController();
  const t = setTimeout(() => ac.abort(), timeoutMs);
  try {
    const res = await fetch(url, {
      method: 'GET',
      headers: { 'user-agent': UA, accept: 'text/html,*/*' },
      redirect: 'follow',
      signal: ac.signal,
    });
    if (!res.ok) throw new Error(`HTTP ${res.status}`);
    const finalUrl = res.url || url;
    if (finalUrl !== url) {
      try { await assertPublicHost(new URL(finalUrl).hostname); } catch { throw new Error('redirect to non-public host'); }
    }
    // Capture iframe-embedability so the website-analysis result page can show a
    // "preview blocked — open in new tab" placeholder instead of a blank white
    // iframe when X-Frame-Options/CSP frame-ancestors disallow embedding.
    const xfo = (res.headers.get('x-frame-options') || '').toLowerCase();
    const csp = (res.headers.get('content-security-policy') || '').toLowerCase();
    const fa = csp.match(/frame-ancestors\s+([^;]+)/);
    const faVal = fa ? fa[1].trim() : '';
    let iframeBlocked = false;
    if (xfo.includes('deny') || xfo.includes('sameorigin')) iframeBlocked = true;
    if (faVal && !faVal.includes('*')) iframeBlocked = true; // anything other than wildcard blocks us cross-origin
    const body = await res.text();
    return { html: body, url: finalUrl, status: res.status, iframe_blocked: !!iframeBlocked };
  } finally {
    clearTimeout(t);
  }
}

function abs(base, href) {
  try { return new URL(href, base).toString(); } catch { return null; }
}

function extractHexColors(text, limit = 32) {
  // Match #abc, #abcdef, rgb(...) — but normalize to hex-ish key
  const seen = new Map();
  const re = /#([0-9a-fA-F]{3}|[0-9a-fA-F]{6})\b/g;
  let m;
  while ((m = re.exec(text)) && seen.size < limit) {
    let h = m[1].toLowerCase();
    if (h.length === 3) h = h.split('').map(c => c + c).join('');
    const key = '#' + h;
    seen.set(key, (seen.get(key) || 0) + 1);
  }
  // Sort by frequency descending
  return [...seen.entries()].sort((a, b) => b[1] - a[1]).map(([c]) => c);
}

function extractPhone(text) {
  // tel: first — that's the only unambiguous signal. Then look for an area-code
  // pattern that's clearly a phone (parens around area code OR explicit
  // `-`/`.` separator). The previous fallback matched any three groups of digits
  // separated by whitespace — timestamps, image dimensions, and CSS values
  // (`200 300 4000`) all qualified.
  const tel = text.match(/tel:([+\d\-\s().]{7,})/i);
  if (tel) return tel[1].trim();
  const m = text.match(/(?:\+?1[\s.\-]?)?(?:\((\d{3})\)\s*|(\d{3})[.\-])(\d{3})[.\-](\d{4})/);
  if (m) {
    const area = m[1] || m[2];
    return `(${area}) ${m[3]}-${m[4]}`;
  }
  return null;
}

function extractEmail(text) {
  const m = text.match(/mailto:([^"'>\s]+)/i);
  if (m) return m[1].trim();
  const e = text.match(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/);
  return e ? e[0] : null;
}

export async function scrapeWebsiteMeta(url) {
  if (!/^https?:\/\//i.test(url)) url = 'https://' + url;
  const { html, iframe_blocked } = await fetchHtml(url);
  const $ = cheerio.load(html);

  const meta = (name) =>
    $(`meta[property="${name}"]`).attr('content') ||
    $(`meta[name="${name}"]`).attr('content') ||
    null;

  const title = meta('og:title') || $('title').first().text().trim() || null;
  const description = meta('og:description') || meta('description') || meta('twitter:description') || null;
  const siteName = meta('og:site_name') || null;

  let heroImage = meta('og:image') || meta('twitter:image') || null;
  if (heroImage) heroImage = abs(url, heroImage);
  if (!heroImage) {
    // first reasonably-large <img> — must declare width >=400 AND height >=300.
    // (Previous predicate inverted the truthiness so that a missing width/height
    // accepted 1x1 tracking pixels as the hero.)
    $('img').each((_, el) => {
      if (heroImage) return false;
      const src = $(el).attr('src');
      if (!src) return;
      const w = parseInt($(el).attr('width') || '0', 10);
      const h = parseInt($(el).attr('height') || '0', 10);
      if (w >= 400 && h >= 300) {
        heroImage = abs(url, src);
        return false;
      }
    });
  }

  // Inline-CSS palette (don't fetch external CSS for now — keep it light + safe)
  const inlineCss = $('style').map((_, el) => $(el).text()).get().join('\n');
  const palette = extractHexColors(inlineCss + '\n' + html);

  // Phone + email — search visible text + tel/mailto links
  const visibleText = $('body').text().replace(/\s+/g, ' ');
  const phone = extractPhone(html) || extractPhone(visibleText);
  const email = extractEmail(html) || extractEmail(visibleText);

  return {
    url,
    title,
    description,
    site_name: siteName,
    hero_image: heroImage,
    palette,
    color_primary: palette[0] || null,
    color_accent: palette[1] || null,
    phone,
    email,
    iframe_blocked: !!iframe_blocked,
  };
}