← back to Directory Core

test/compliance.test.ts

196 lines

// test/compliance.test.ts
// Tests for src/compliance.ts — robots.txt gating, rate limiting, fetchCompliant.
//
// Strategy: undici's fetch (used internally by compliance.ts) honours the global
// dispatcher. We swap in a MockAgent before each test and restore the real Agent
// after, so every fetch is fully intercepted without hitting the network.
//
// DATABASE_URL must be set before auth.ts (imported transitively via db.ts) loads.

import { describe, it, before, after, beforeEach, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import {
  MockAgent,
  setGlobalDispatcher,
  getGlobalDispatcher,
  Agent,
} from 'undici';

// Satisfy db.ts module-load guard (imported transitively through auth.ts if needed)
process.env.DATABASE_URL = process.env.DATABASE_URL || 'postgresql://fake:fake@localhost:5432/fake_test';

// ── Import the module under test ─────────────────────────────────────────────
// compliance.ts is a pure ESM module; we import it once. The internal
// `robotsCache` and rate-limit maps persist between tests within a single run.
// We work around cache persistence by using distinct hostnames per test group.

import { isAllowed, setHostRateLimit, fetchCompliant } from '../src/compliance.js';

// ── Helpers ───────────────────────────────────────────────────────────────────

function makeMockAgent() {
  const agent = new MockAgent();
  agent.disableNetConnect(); // fail fast on any un-intercepted request
  setGlobalDispatcher(agent);
  return agent;
}

// ── isAllowed ─────────────────────────────────────────────────────────────────

describe('isAllowed — robots.txt parsing', () => {
  const HOST = 'http://robots-test-unique.example';
  let agent: MockAgent;
  let realDispatcher: ReturnType<typeof getGlobalDispatcher>;

  before(() => {
    realDispatcher = getGlobalDispatcher();
    agent = makeMockAgent();

    // robots.txt for this host: Disallow /private/*
    const robotsTxt = [
      'User-agent: *',
      'Disallow: /private/',
    ].join('\n');

    agent
      .get(HOST)
      .intercept({ path: '/robots.txt', method: 'GET' })
      .reply(200, robotsTxt, { headers: { 'content-type': 'text/plain' } });
  });

  after(async () => {
    setGlobalDispatcher(realDispatcher);
    await agent.close();
  });

  it('allows a public path not in Disallow rules', async () => {
    const allowed = await isAllowed(`${HOST}/public/page`);
    assert.equal(allowed, true);
  });

  it('disallows a path matching Disallow: /private/', async () => {
    // robots cache is warm from the first call above — no second network hit needed
    const allowed = await isAllowed(`${HOST}/private/secret`);
    assert.equal(allowed, false);
  });
});

// ── setHostRateLimit + gateHost (via fetchCompliant timing) ───────────────────

describe('setHostRateLimit — enforces minimum inter-request gap', () => {
  // Use 10 rps → minInterval = 100ms. We make two consecutive calls and assert
  // the elapsed wall time is at least 100ms.
  const HOST = 'http://ratelimit-unique.example';
  let agent: MockAgent;
  let realDispatcher: ReturnType<typeof getGlobalDispatcher>;

  before(() => {
    realDispatcher = getGlobalDispatcher();
    agent = makeMockAgent();
    setHostRateLimit(HOST.replace('http://', ''), 10); // 10 rps → 100ms gap

    // robots.txt — allow everything so gating is the only delay
    agent
      .get(HOST)
      .intercept({ path: '/robots.txt', method: 'GET' })
      .reply(200, 'User-agent: *\nAllow: /', { headers: { 'content-type': 'text/plain' } });

    // Two page fetches
    agent
      .get(HOST)
      .intercept({ path: '/page1', method: 'GET' })
      .reply(200, 'page1');

    agent
      .get(HOST)
      .intercept({ path: '/page2', method: 'GET' })
      .reply(200, 'page2');
  });

  after(async () => {
    setGlobalDispatcher(realDispatcher);
    await agent.close();
  });

  it('enforces >= 100ms gap between two requests to same host at 10 rps', async () => {
    const t0 = Date.now();
    await fetchCompliant(`${HOST}/page1`, { respectRobots: false });
    await fetchCompliant(`${HOST}/page2`, { respectRobots: false });
    const elapsed = Date.now() - t0;
    assert.ok(elapsed >= 100, `Expected >= 100ms gap, got ${elapsed}ms`);
  });
});

// ── fetchCompliant — CAPTCHA detection ───────────────────────────────────────

describe('fetchCompliant — throws CAPTCHA_DETECTED on 403 with captcha body', () => {
  const HOST = 'http://captcha-unique.example';
  let agent: MockAgent;
  let realDispatcher: ReturnType<typeof getGlobalDispatcher>;

  before(() => {
    realDispatcher = getGlobalDispatcher();
    agent = makeMockAgent();

    agent
      .get(HOST)
      .intercept({ path: '/gated', method: 'GET' })
      .reply(403, '<html>please complete the hcaptcha challenge</html>', {
        headers: { 'content-type': 'text/html' },
      });
  });

  after(async () => {
    setGlobalDispatcher(realDispatcher);
    await agent.close();
  });

  it('throws an error with code=CAPTCHA_DETECTED', async () => {
    await assert.rejects(
      () => fetchCompliant(`${HOST}/gated`, { respectRobots: false }),
      (err: any) => {
        assert.equal(err.code, 'CAPTCHA_DETECTED');
        return true;
      },
    );
  });
});

// ── fetchCompliant — 500 then 200 retry ──────────────────────────────────────

describe('fetchCompliant — retries once on 500 and succeeds on 200', () => {
  const HOST = 'http://retry-unique.example';
  let agent: MockAgent;
  let realDispatcher: ReturnType<typeof getGlobalDispatcher>;
  let attemptCount = 0;

  before(() => {
    realDispatcher = getGlobalDispatcher();
    agent = makeMockAgent();

    // First call → 500; second call → 200.
    agent
      .get(HOST)
      .intercept({ path: '/flaky', method: 'GET' })
      .reply(() => {
        attemptCount++;
        if (attemptCount === 1) {
          return { statusCode: 500, data: 'Server Error', responseOptions: { headers: { 'content-type': 'text/plain' } } };
        }
        return { statusCode: 200, data: 'OK', responseOptions: { headers: { 'content-type': 'text/plain' } } };
      })
      .times(2); // intercept up to 2 times
  });

  after(async () => {
    setGlobalDispatcher(realDispatcher);
    await agent.close();
  });

  it('retries after 500 and ultimately returns 200', async () => {
    const res = await fetchCompliant(`${HOST}/flaky`, { respectRobots: false });
    assert.equal(res.status, 200, `Expected 200 after retry, got ${res.status}`);
    assert.equal(attemptCount, 2, `Expected 2 attempts (1 retry), got ${attemptCount}`);
  });
});