`lib/compliance/fetch-guard.js`

248 lines
'use strict';
/**
 * Fetch guard — SSRF protection for EVERY outbound fetch (spec §32, §6.4).
 *
 * Guarantees:
 *   - LinkedIn hosts are hard-blocked (§6.3/§6.4) — the app NEVER fetches them.
 *   - No fetch may resolve to a private / loopback / link-local / metadata IP
 *     (§32 "DNS/IP checks blocking private, loopback, metadata, and link-local").
 *   - Automated public web research is OFF unless explicitly enabled by env
 *     (ALLOW_AUTOMATED_PUBLIC_WEB_RESEARCH === 'true').
 *   - A descriptive, admin-contact User-Agent is always sent (§23).
 *   - Per-host token bucket enforces a minimum delay / max requests per minute.
 *   - Redirects are re-checked against the same guard (no redirect to a blocked
 *     host or private IP).
 */

const dns = require('dns').promises;
const T = require('../types');

// ---------------------------------------------------------------------------
// IP range checks
// ---------------------------------------------------------------------------

/** Parse an IPv4 "a.b.c.d" → 32-bit unsigned int, or null if not IPv4. */
function ipv4ToInt(ip) {
  const m = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/.exec(ip);
  if (!m) return null;
  const parts = m.slice(1).map(Number);
  if (parts.some((p) => p > 255)) return null;
  return ((parts[0] << 24) >>> 0) + (parts[1] << 16) + (parts[2] << 8) + parts[3];
}

function inCidr(ipInt, netStr, bits) {
  const net = ipv4ToInt(netStr);
  if (net == null || ipInt == null) return false;
  const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
  return (ipInt & mask) === (net & mask);
}

/**
 * isBlockedIp(ip) — true for any private / loopback / link-local / metadata /
 * unspecified address (§32). Covers both IPv4 and the IPv6 forms we care about.
 */
function isBlockedIp(ip) {
  if (!ip || typeof ip !== 'string') return true; // fail closed
  const addr = ip.trim().toLowerCase().replace(/^\[|\]$/g, '');

  // --- IPv6 ---
  if (addr.includes(':')) {
    if (addr === '::' || addr === '::0' || addr === '0:0:0:0:0:0:0:0') return true; // unspecified
    if (addr === '::1' || addr === '0:0:0:0:0:0:0:1') return true;                   // loopback
    // fc00::/7  — unique local (fc.. / fd..)
    if (/^f[cd][0-9a-f]{0,2}:/.test(addr)) return true;
    // fe80::/10 — link-local
    if (/^fe[89ab][0-9a-f]?:/.test(addr) || /^fe80:/.test(addr)) return true;
    // IPv4-mapped ::ffff:a.b.c.d — unwrap and re-check as IPv4
    const mapped = /^::ffff:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(addr);
    if (mapped) return isBlockedIp(mapped[1]);
    return false; // other global IPv6 — allowed
  }

  // --- IPv4 ---
  const n = ipv4ToInt(addr);
  if (n == null) return true; // not a parseable IPv4 → fail closed

  if (inCidr(n, '10.0.0.0', 8)) return true;        // private
  if (inCidr(n, '172.16.0.0', 12)) return true;     // private
  if (inCidr(n, '192.168.0.0', 16)) return true;    // private
  if (inCidr(n, '127.0.0.0', 8)) return true;       // loopback
  if (inCidr(n, '169.254.0.0', 16)) return true;    // link-local (incl. 169.254.169.254 metadata)
  if (inCidr(n, '0.0.0.0', 8)) return true;         // "this" network / 0.0.0.0
  if (inCidr(n, '100.64.0.0', 10)) return true;     // CGNAT (also used by tailnets) — treat as internal
  return false;
}

/** True when host === blocked OR endsWith .blocked (subdomain) — §6.4. */
function isLinkedInHost(host) {
  if (!host) return false;
  const h = host.toLowerCase();
  return T.LINKEDIN_BLOCKED_HOSTS.some((b) => h === b.toLowerCase() || h.endsWith(`.${b.toLowerCase()}`));
}

// ---------------------------------------------------------------------------
// assertFetchAllowed
// ---------------------------------------------------------------------------

/**
 * assertFetchAllowed(url) — throws unless the URL is safe to fetch.
 * Rejects: non-http(s), LinkedIn hosts, and hosts that resolve (ANY A/AAAA
 * record) to a blocked IP range. Resolves DNS via dns.promises.lookup(all).
 */
async function assertFetchAllowed(url) {
  let u;
  try {
    u = new URL(url);
  } catch (_e) {
    throw new Error(`fetch-guard: invalid URL "${url}"`);
  }

  if (u.protocol !== 'http:' && u.protocol !== 'https:') {
    throw new Error(`fetch-guard: protocol "${u.protocol}" not allowed (http/https only)`);
  }

  const host = u.hostname.toLowerCase();

  // §6.4 — LinkedIn hard block.
  if (isLinkedInHost(host)) {
    throw new Error(`fetch-guard: LinkedIn host "${host}" is blocked from automated fetching (§6.3/§6.4)`);
  }

  // If the host is already a literal IP, check it directly.
  const literal = host.replace(/^\[|\]$/g, '');
  if (ipv4ToInt(literal) != null || literal.includes(':')) {
    if (isBlockedIp(literal)) {
      throw new Error(`fetch-guard: host resolves to blocked IP "${literal}" (private/loopback/metadata/link-local) (§32)`);
    }
    return { host, addresses: [literal] };
  }

  // Resolve ALL addresses; block if ANY is private (DNS-rebinding defense).
  let addresses;
  try {
    addresses = await dns.lookup(host, { all: true });
  } catch (e) {
    throw new Error(`fetch-guard: DNS lookup failed for "${host}": ${e.message}`);
  }
  if (!addresses || addresses.length === 0) {
    throw new Error(`fetch-guard: "${host}" resolved to no addresses`);
  }
  for (const a of addresses) {
    if (isBlockedIp(a.address)) {
      throw new Error(`fetch-guard: "${host}" resolves to blocked IP "${a.address}" (§32)`);
    }
  }
  return { host, addresses: addresses.map((a) => a.address) };
}

// ---------------------------------------------------------------------------
// Per-host token bucket (min delay / max requests per minute)
// ---------------------------------------------------------------------------

const buckets = new Map(); // host -> { lastAt, timestamps: number[] }

function defaultRpm() {
  const v = Number(process.env.DEFAULT_REQUESTS_PER_MINUTE || '6');
  return Number.isFinite(v) && v > 0 ? v : 6;
}

async function throttleHost(host, { maxRequestsPerMinute, minimumDelayMs } = {}) {
  const rpm = maxRequestsPerMinute || defaultRpm();
  const minDelay = minimumDelayMs != null ? minimumDelayMs : Math.ceil(60000 / rpm);
  const now = Date.now();
  let b = buckets.get(host);
  if (!b) {
    b = { lastAt: 0, timestamps: [] };
    buckets.set(host, b);
  }
  // Purge timestamps older than 60s.
  b.timestamps = b.timestamps.filter((t) => now - t < 60000);

  // Enforce max-requests-per-minute.
  if (b.timestamps.length >= rpm) {
    const waitMs = 60000 - (now - b.timestamps[0]);
    if (waitMs > 0) await sleep(waitMs);
  }
  // Enforce minimum inter-request delay.
  const since = Date.now() - b.lastAt;
  if (b.lastAt && since < minDelay) await sleep(minDelay - since);

  const stamp = Date.now();
  b.lastAt = stamp;
  b.timestamps.push(stamp);
}

function sleep(ms) {
  return new Promise((r) => setTimeout(r, ms));
}

/** Test/reset helper — clears the in-memory rate-limit state. */
function _resetBuckets() {
  buckets.clear();
}

// ---------------------------------------------------------------------------
// safeFetch
// ---------------------------------------------------------------------------

/**
 * safeFetch(url, opts) — the ONLY sanctioned outbound HTTP entrypoint.
 *  - refuses unless ALLOW_AUTOMATED_PUBLIC_WEB_RESEARCH === 'true'
 *  - runs assertFetchAllowed (LinkedIn + SSRF)
 *  - sets a descriptive User-Agent from CRAWLER_USER_AGENT
 *  - throttles per host (token bucket)
 *  - follows redirects MANUALLY, re-checking each hop with assertFetchAllowed
 *
 * opts: { maxRequestsPerMinute, minimumDelayMs, maxRedirects, headers, ...fetchOpts }
 */
async function safeFetch(url, opts = {}) {
  if (process.env.ALLOW_AUTOMATED_PUBLIC_WEB_RESEARCH !== 'true') {
    throw new Error(
      'safeFetch: automated public web research is DISABLED. Set ALLOW_AUTOMATED_PUBLIC_WEB_RESEARCH=true to enable (spec §30).'
    );
  }

  const ua =
    process.env.CRAWLER_USER_AGENT ||
    'RENTV-Advertiser-Research/1.0 (+admin@rentv.com)';
  const maxRedirects = opts.maxRedirects != null ? opts.maxRedirects : 5;

  let current = url;
  for (let hop = 0; hop <= maxRedirects; hop++) {
    // Guard EVERY hop (initial + each redirect target).
    const { host } = await assertFetchAllowed(current);
    await throttleHost(host, opts);

    const headers = Object.assign({ 'user-agent': ua }, opts.headers || {});
    const fetchOpts = Object.assign({}, opts, {
      headers,
      redirect: 'manual', // we re-check redirects ourselves
    });
    delete fetchOpts.maxRedirects;
    delete fetchOpts.maxRequestsPerMinute;
    delete fetchOpts.minimumDelayMs;

    const res = await fetch(current, fetchOpts);

    // Redirect? re-check the Location target through the guard.
    if (res.status >= 300 && res.status < 400 && res.headers.get('location')) {
      if (hop === maxRedirects) {
        throw new Error(`safeFetch: too many redirects (>${maxRedirects}) from "${url}"`);
      }
      const next = new URL(res.headers.get('location'), current).toString();
      current = next; // loop re-guards it (blocks redirect-to-LinkedIn / private IP)
      continue;
    }
    return res;
  }
  throw new Error(`safeFetch: redirect loop exhausted for "${url}"`);
}

module.exports = {
  assertFetchAllowed,
  isBlockedIp,
  isLinkedInHost,
  safeFetch,
  _resetBuckets,
};