`src/lib/compliance.ts`

112 lines
import 'dotenv/config';
import { fetch } from 'undici';
// @ts-ignore — robots-parser has no types ship
import robotsParser from 'robots-parser';

const USER_AGENT = process.env.USER_AGENT
  || 'LawyerDirectoryBuilder/0.1 (research; contact: steveabramsdesigns@gmail.com)';
const DEFAULT_RPS = parseFloat(process.env.STATE_BAR_RPS || '0.5');

const robotsCache = new Map<string, { robots: any; fetchedAt: number }>();
const ONE_DAY = 24 * 60 * 60 * 1000;

async function getRobotsFor(urlStr: string) {
  const u = new URL(urlStr);
  const cached = robotsCache.get(u.host);
  if (cached && Date.now() - cached.fetchedAt < ONE_DAY) return cached.robots;

  const robotsUrl = `${u.protocol}//${u.host}/robots.txt`;
  let body = '';
  try {
    const res = await fetch(robotsUrl, {
      headers: { 'User-Agent': USER_AGENT },
      signal: AbortSignal.timeout(10000),
    });
    if (res.ok) body = await res.text();
  } catch { /* permissive on transport error */ }

  const robots = robotsParser(robotsUrl, body);
  robotsCache.set(u.host, { robots, fetchedAt: Date.now() });
  return robots;
}

export async function isAllowed(urlStr: string): Promise<boolean> {
  const robots = await getRobotsFor(urlStr);
  const verdict = robots.isAllowed(urlStr, USER_AGENT);
  return verdict !== false;
}

const lastFetchByHost = new Map<string, number>();
const rpsByHost = new Map<string, number>();

export function setHostRateLimit(host: string, rps: number) {
  rpsByHost.set(host, rps);
}

async function gateHost(host: string) {
  const rps = rpsByHost.get(host) ?? DEFAULT_RPS;
  const minInterval = 1000 / rps;
  const last = lastFetchByHost.get(host) || 0;
  const wait = Math.max(0, last + minInterval - Date.now());
  if (wait > 0) await new Promise(r => setTimeout(r, wait));
  lastFetchByHost.set(host, Date.now());
}

export interface FetchOpts {
  respectRobots?: boolean;
  accept?: string;
  headers?: Record<string, string>;
  timeoutMs?: number;
}

export async function fetchCompliant(urlStr: string, opts: FetchOpts = {}) {
  const u = new URL(urlStr);
  if (opts.respectRobots !== false) {
    const allowed = await isAllowed(urlStr);
    if (!allowed) {
      const err = new Error(`robots.txt disallows ${urlStr}`) as Error & { code?: string };
      err.code = 'ROBOTS_DISALLOWED';
      throw err;
    }
  }
  await gateHost(u.host);

  const headers: Record<string, string> = {
    'User-Agent': USER_AGENT,
    Accept: opts.accept || 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.9',
    ...(opts.headers || {}),
  };

  let attempt = 0;
  while (true) {
    attempt++;
    let res: Response | undefined;
    try {
      res = await fetch(urlStr, { headers, signal: AbortSignal.timeout(opts.timeoutMs || 30000) }) as unknown as Response;
    } catch (e) {
      if (attempt >= 4) throw e;
      await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
      continue;
    }

    if (res.status === 403 || res.status === 401) {
      const text = await res.text();
      if (/cf-challenge|recaptcha|hcaptcha|captcha/i.test(text)) {
        const err = new Error(`CAPTCHA detected at ${urlStr} — aborting per policy`) as Error & { code?: string };
        err.code = 'CAPTCHA_DETECTED';
        throw err;
      }
      // re-wrap response with body since we consumed it
      return new Response(text, { status: res.status, headers: res.headers });
    }

    if (res.status >= 500 && attempt < 4) {
      await new Promise(r => setTimeout(r, 1000 * Math.pow(2, attempt - 1)));
      continue;
    }

    return res;
  }
}