`agents/shared/compliance.js`

165 lines
/**
 * Professional Directory — Compliance utilities.
 *
 * Every crawler MUST go through fetchCompliant() so we get:
 *   1. robots.txt enforcement (per-host cache, 24 h TTL)
 *   2. per-host rate limit (default 0.5 rps; configurable per source)
 *   3. consistent User-Agent string with contact info
 *   4. exponential backoff on 4xx/5xx
 *   5. opt-out filter helpers for downstream API output
 */
const path = require('path');
require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
const { fetch } = require('undici');
const robotsParser = require('robots-parser');

const USER_AGENT = process.env.USER_AGENT
  || 'ProfessionalDirectoryBot/0.1 (research; contact: steveabramsdesigns@gmail.com)';
const DEFAULT_RPS = parseFloat(process.env.DEFAULT_RATE_LIMIT_RPS || '0.5');

// ─── Per-host robots.txt cache ───────────────────────────────────────────────

const robotsCache = new Map(); // protocol+host -> { robots, fetchedAt, denied }
const ONE_DAY = 24 * 60 * 60 * 1000;

async function getRobotsFor(urlStr) {
  const u = new URL(urlStr);
  // Cache key includes scheme — robots.txt is per-scheme per RFC 9309.
  const cacheKey = `${u.protocol}//${u.host}`;
  const cached = robotsCache.get(cacheKey);
  if (cached && (Date.now() - cached.fetchedAt) < ONE_DAY) return cached;

  const robotsUrl = `${cacheKey}/robots.txt`;
  let body = '';
  let denied = false;
  try {
    const res = await fetch(robotsUrl, {
      headers: { 'User-Agent': USER_AGENT },
      signal: AbortSignal.timeout(10000),
    });
    if (res.ok) {
      body = await res.text();
    } else if (res.status >= 500) {
      // RFC 9309: 5xx on robots.txt MUST be treated as full disallow.
      denied = true;
    }
    // 4xx (incl. 404) is permissive per RFC.
  } catch (_) { /* transport error → permissive */ }

  const robots = robotsParser(robotsUrl, body);
  const entry = { robots, fetchedAt: Date.now(), denied };
  robotsCache.set(cacheKey, entry);
  return entry;
}

async function isAllowed(urlStr) {
  const { robots, denied } = await getRobotsFor(urlStr);
  if (denied) return false;
  return robots.isAllowed(urlStr, USER_AGENT) !== false;
}

// ─── Per-host rate limiter (token-bucket-ish, simple gating) ─────────────────
// Keys are normalized via hostKey() so callers may pass either u.host or
// u.hostname interchangeably without silently bypassing the limit.

const lastFetchByHost = new Map(); // host -> ms timestamp (next-allowed)
const rpsByHost = new Map();        // host -> rps override

function hostKey(hostOrUrl) {
  if (typeof hostOrUrl !== 'string') return String(hostOrUrl);
  // strip :port; lowercase
  const lower = hostOrUrl.toLowerCase();
  const colon = lower.indexOf(':');
  return colon === -1 ? lower : lower.slice(0, colon);
}

function setHostRateLimit(host, rps) { rpsByHost.set(hostKey(host), rps); }

async function gateHost(host) {
  const k = hostKey(host);
  const rps = rpsByHost.get(k) ?? DEFAULT_RPS;
  const minIntervalMs = 1000 / rps;
  const now = Date.now();
  const last = lastFetchByHost.get(k) || 0;
  // Reserve the slot BEFORE awaiting so concurrent callers serialize against
  // each other rather than all reading the same `last` and racing.
  const reservedNext = Math.max(now, last) + minIntervalMs;
  lastFetchByHost.set(k, reservedNext);
  const wait = Math.max(0, last - now);
  if (wait > 0) await new Promise(r => setTimeout(r, wait));
}

// ─── Public fetch wrapper ────────────────────────────────────────────────────

async function fetchCompliant(urlStr, opts = {}) {
  const u = new URL(urlStr);

  if (opts.respectRobots !== false) {
    const allowed = await isAllowed(urlStr);
    if (!allowed) {
      const err = new Error(`robots.txt disallows ${urlStr}`);
      err.code = 'ROBOTS_DISALLOWED';
      throw err;
    }
  }

  await gateHost(u.hostname);

  const headers = {
    'User-Agent': USER_AGENT,
    Accept: opts.accept || 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    ...(opts.headers || {}),
  };

  let attempt = 0;
  // Exponential backoff: 1s, 2s, 4s, then give up.
  while (true) {
    attempt++;
    let res;
    try {
      res = await fetch(urlStr, {
        headers,
        method: opts.method || 'GET',
        body: opts.body,
        redirect: opts.redirect || 'follow',
        signal: AbortSignal.timeout(opts.timeoutMs || 30000),
      });
    } catch (e) {
      if (attempt >= 4) throw e;
      await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
      continue;
    }

    // CAPTCHA / Cloudflare detection — DO NOT bypass; abort the source.
    if (res.status === 403 || res.status === 401) {
      const text = await res.text().catch(() => '');
      if (/cf-challenge|recaptcha|hcaptcha|captcha/i.test(text)) {
        const err = new Error(`CAPTCHA / challenge detected at ${urlStr} — aborting per compliance policy`);
        err.code = 'CAPTCHA_DETECTED';
        err.body = text.slice(0, 4000);
        throw err;
      }
    }

    if (res.status >= 500 && attempt < 4) {
      await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
      continue;
    }

    return res;
  }
}

// ─── Opt-out helpers ─────────────────────────────────────────────────────────

const OPT_OUT_WHERE = 'opted_out = false';

module.exports = {
  USER_AGENT,
  DEFAULT_RPS,
  fetchCompliant,
  isAllowed,
  setHostRateLimit,
  OPT_OUT_WHERE,
};