`src/compliance.ts`

197 lines
// directory-core / compliance
//
// Polite HTTP fetching for vendor / public-records crawlers across verticals:
//   - robots.txt parsing + 24h cache (per host)
//   - per-host rate limiting (configurable, default 0.5 rps)
//   - retry on 5xx with exponential backoff (4 attempts max)
//   - 403/401 CAPTCHA detection — throws CAPTCHA_DETECTED rather than retrying
//     in a loop (Steve's rule: never burn through hCaptcha/reCAPTCHA budgets)
//
// Source: extracted from `lawyer-directory-builder/src/lib/compliance.ts` 2026-05-04.
// Already vertical-neutral; copied verbatim with the consumer-facing
// USER_AGENT default updated to identify directory-core rather than the
// lawyer-specific source. Each consumer can override via env.

import 'dotenv/config';
import { fetch } from 'undici';
import dns from 'node:dns/promises';
// robots-parser ships incomplete types — its default export is a callable
// factory, but the published .d.ts shows a namespace. Cast through unknown.
import robotsParserImport from 'robots-parser';
const robotsParser = robotsParserImport as unknown as (url: string, body: string) => {
  isAllowed(url: string, ua: string): boolean | undefined;
};
// Runtime sanity check (audit P2 2026-05-04): if robots-parser ever flips its
// ESM/CJS shape, the cast above silently breaks. Verify at module load.
if (typeof robotsParser !== 'function') {
  throw new Error(
    'directory-core/compliance: robots-parser default export is not callable — package shape changed?'
  );
}

const USER_AGENT = process.env.USER_AGENT
  || 'directory-core/0.2 (research; contact: steveabramsdesigns@gmail.com)';
const DEFAULT_RPS = parseFloat(process.env.CRAWLER_RPS || process.env.STATE_BAR_RPS || '0.5');

// ── SSRF defense (audit P1-b 2026-05-04) ──────────────────────────────────
//
// fetchCompliant accepts arbitrary URLs from callers. Today every caller
// passes hard-coded vendor URLs, but the moment any consumer threads a
// user-supplied URL through (admin form, OAuth callback, etc.), unguarded
// fetch becomes an SSRF + cloud-metadata-credentials-exfil pivot. Block at
// the perimeter: resolve the hostname's A/AAAA records and reject if any
// fall in private/loopback/link-local/CGNAT/cloud-metadata ranges.
//
// Same gate also applies to `getRobotsFor` (it fetches /robots.txt before
// the host check, so its fetch is also a free SSRF hop).
function isPrivateAddr(addr: string): boolean {
  // IPv4 private + special ranges
  if (/^10\./.test(addr)) return true;                          // 10.0.0.0/8
  if (/^192\.168\./.test(addr)) return true;                    // 192.168.0.0/16
  if (/^172\.(1[6-9]|2\d|3[0-1])\./.test(addr)) return true;    // 172.16.0.0/12
  if (/^127\./.test(addr)) return true;                         // 127.0.0.0/8 loopback
  if (/^169\.254\./.test(addr)) return true;                    // 169.254.0.0/16 link-local + cloud metadata
  if (/^100\.(6[4-9]|[7-9]\d|1[0-1]\d|12[0-7])\./.test(addr)) return true; // 100.64.0.0/10 CGNAT
  if (addr === '0.0.0.0') return true;
  // IPv6
  if (addr === '::1' || addr === '::') return true;
  if (/^fe80:/i.test(addr)) return true;                         // link-local
  if (/^fc[0-9a-f]{2}:|^fd[0-9a-f]{2}:/i.test(addr)) return true; // unique-local
  return false;
}

async function assertPublicHost(urlStr: string): Promise<void> {
  const u = new URL(urlStr);
  // Resolve all A and AAAA records — reject if ANY is private. Using
  // Promise.allSettled so a missing AAAA record (common) doesn't fail the
  // whole check.
  const [a, aaaa] = await Promise.allSettled([
    dns.resolve4(u.hostname),
    dns.resolve6(u.hostname),
  ]);
  const addrs: string[] = [];
  if (a.status === 'fulfilled') addrs.push(...a.value);
  if (aaaa.status === 'fulfilled') addrs.push(...aaaa.value);
  // If DNS resolution returned nothing for both families, the host is
  // unresolvable — let the fetch fail naturally rather than mis-classify.
  if (addrs.length === 0) return;
  for (const addr of addrs) {
    if (isPrivateAddr(addr)) {
      const err = new Error(`SSRF blocked — ${u.hostname} resolves to private address ${addr}`) as Error & { code?: string };
      err.code = 'SSRF_BLOCKED';
      throw err;
    }
  }
}

const robotsCache = new Map<string, { robots: any; fetchedAt: number }>();
const ONE_DAY = 24 * 60 * 60 * 1000;

async function getRobotsFor(urlStr: string) {
  const u = new URL(urlStr);
  const cached = robotsCache.get(u.host);
  if (cached && Date.now() - cached.fetchedAt < ONE_DAY) return cached.robots;

  const robotsUrl = `${u.protocol}//${u.host}/robots.txt`;
  let body = '';
  try {
    // SSRF gate (audit P1-b 2026-05-04) — robots.txt fetch is its own attack
    // surface; gate it the same way as fetchCompliant.
    await assertPublicHost(robotsUrl);
    const res = await fetch(robotsUrl, {
      headers: { 'User-Agent': USER_AGENT },
      signal: AbortSignal.timeout(10000),
    });
    if (res.ok) body = await res.text();
  } catch { /* permissive on transport error AND on SSRF reject */ }

  const robots = robotsParser(robotsUrl, body);
  robotsCache.set(u.host, { robots, fetchedAt: Date.now() });
  return robots;
}

export async function isAllowed(urlStr: string): Promise<boolean> {
  const robots = await getRobotsFor(urlStr);
  const verdict = robots.isAllowed(urlStr, USER_AGENT);
  return verdict !== false;
}

const lastFetchByHost = new Map<string, number>();
const rpsByHost = new Map<string, number>();

export function setHostRateLimit(host: string, rps: number) {
  rpsByHost.set(host, rps);
}

async function gateHost(host: string) {
  const rps = rpsByHost.get(host) ?? DEFAULT_RPS;
  const minInterval = 1000 / rps;
  const last = lastFetchByHost.get(host) || 0;
  const wait = Math.max(0, last + minInterval - Date.now());
  if (wait > 0) await new Promise(r => setTimeout(r, wait));
  lastFetchByHost.set(host, Date.now());
}

export interface FetchOpts {
  respectRobots?: boolean;
  accept?: string;
  headers?: Record<string, string>;
  timeoutMs?: number;
}

export async function fetchCompliant(urlStr: string, opts: FetchOpts = {}) {
  // SSRF gate FIRST — before robots.txt, before rate-limit, before fetch.
  // Audit P1-b 2026-05-04: blocks RFC1918, loopback, link-local, CGNAT, and
  // cloud-metadata (169.254.169.254). Throws SSRF_BLOCKED on reject.
  await assertPublicHost(urlStr);

  const u = new URL(urlStr);
  if (opts.respectRobots !== false) {
    const allowed = await isAllowed(urlStr);
    if (!allowed) {
      const err = new Error(`robots.txt disallows ${urlStr}`) as Error & { code?: string };
      err.code = 'ROBOTS_DISALLOWED';
      throw err;
    }
  }
  await gateHost(u.host);

  const headers: Record<string, string> = {
    'User-Agent': USER_AGENT,
    Accept: opts.accept || 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.9',
    ...(opts.headers || {}),
  };

  let attempt = 0;
  while (true) {
    attempt++;
    let res: Response | undefined;
    try {
      res = await fetch(urlStr, { headers, signal: AbortSignal.timeout(opts.timeoutMs || 30000) }) as unknown as Response;
    } catch (e) {
      if (attempt >= 4) throw e;
      await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
      continue;
    }

    if (res.status === 403 || res.status === 401) {
      const text = await res.text();
      // Audit P2 2026-05-04: extended to cover Turnstile, Arkose, Datadome, AWS WAF Captcha.
      if (/cf-challenge|recaptcha|hcaptcha|captcha|turnstile|arkose|datadome|aws-waf-token/i.test(text)) {
        const err = new Error(`CAPTCHA detected at ${urlStr} — aborting per policy`) as Error & { code?: string };
        err.code = 'CAPTCHA_DETECTED';
        throw err;
      }
      return new Response(text, { status: res.status, headers: res.headers });
    }

    if (res.status >= 500 && attempt < 4) {
      await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
      continue;
    }

    return res;
  }
}