`lib/compliance/source-policy.js`

198 lines
'use strict';
/**
 * Source-policy validator — the legal guardrail (spec §6, §9, §10).
 *
 * Enforces the "hard research, privacy, and source rules" in CODE, not docs:
 *   §6.1  no bypassing auth/paywalls/robots/rate-limits/anti-bot
 *   §6.2  no scraping proprietary DBs (CoStar, LoopNet, ZoomInfo, Apollo, MLS)
 *   §6.3  no crawling LinkedIn profile/company pages
 *   §6.4  block automated fetching from LinkedIn hosts in the generic fetcher
 *   §10   the SourcePolicy shape + access-method vocabulary
 *
 * A policy that names a prohibited proprietary DB or a LinkedIn host as its
 * base is ILLEGAL to enable for automation. validateSourcePolicy() flags it;
 * assertSourceEnabledLegal() throws so a pipeline can never run it.
 */

const T = require('../types');

/**
 * Prohibited proprietary databases (§6.2). Login-gated / proprietary — never
 * automate. LinkedIn hosts (§6.3/§6.4) are appended from the shared contract so
 * there is ONE list. MLS is matched by pattern below (any *.mls* / mls.* host).
 */
const PROHIBITED_PROPRIETARY_HOSTS = Object.freeze([
  'costar.com',
  'loopnet.com',
  'zoominfo.com',
  'apollo.io',
  'crexi.com',       // proprietary marketplace DB, login-gated bulk access
  'reonomy.com',     // proprietary property DB
]);

// Full automation blocklist = proprietary DBs + every LinkedIn host (§6.2-6.4).
const PROHIBITED_AUTOMATION_HOSTS = Object.freeze(
  PROHIBITED_PROPRIETARY_HOSTS.concat(T.LINKEDIN_BLOCKED_HOSTS)
);

/** Access methods that, by definition, perform NO automated fetching. */
const NON_AUTOMATED_METHODS = Object.freeze([
  'manual_upload',
  'manual_review_only',
  'authorized_mailbox', // mailbox is authorized-API, not web automation
]);

/** Lowercase, strip scheme/path/port → bare host. Accepts a bare host too. */
function hostOf(urlOrHost) {
  if (!urlOrHost) return '';
  const raw = String(urlOrHost).trim();
  try {
    const u = new URL(raw.includes('://') ? raw : `https://${raw}`);
    return u.hostname.toLowerCase();
  } catch (_e) {
    return raw.toLowerCase().replace(/^\/+/, '').split('/')[0].split(':')[0];
  }
}

/** True when host === blocked OR is a subdomain of blocked (a.b.costar.com). */
function hostMatches(host, blocked) {
  if (!host) return false;
  const h = host.toLowerCase();
  const b = blocked.toLowerCase();
  return h === b || h.endsWith(`.${b}`);
}

/** MLS systems are innumerable; match by pattern (§6.2 "any MLS"). */
function looksLikeMls(host) {
  if (!host) return false;
  const h = host.toLowerCase();
  // mls.<x>, <x>.mls.<y>, <x>mls.<y>, crmls/themls/etc.
  return (
    /(^|\.)mls\./.test(h) ||
    /(^|\.)[a-z]*mls\.[a-z]/.test(h) ||
    /\bmls\b/.test(h.replace(/[.-]/g, ' '))
  );
}

/** Is this host prohibited for AUTOMATED access? (proprietary DB, LinkedIn, MLS) */
function isProhibitedAutomationHost(host) {
  if (!host) return false;
  if (looksLikeMls(host)) return true;
  return PROHIBITED_AUTOMATION_HOSTS.some((b) => hostMatches(host, b));
}

/** §10 — the access method must be one of the seven contract values. */
function accessMethodAllowed(method) {
  return T.SOURCE_ACCESS_METHODS.includes(method);
}

/**
 * validateSourcePolicy(policy) → { valid, errors }
 * Never throws; collects every violation so the /sources UI can show them all.
 */
function validateSourcePolicy(policy) {
  const errors = [];

  if (!policy || typeof policy !== 'object') {
    return { valid: false, errors: ['policy must be an object'] };
  }

  // Required identity fields.
  if (!policy.sourceKey || typeof policy.sourceKey !== 'string') {
    errors.push('sourceKey is required');
  }
  if (!policy.displayName || typeof policy.displayName !== 'string') {
    errors.push('displayName is required');
  }

  // §10 access method vocabulary.
  if (!accessMethodAllowed(policy.accessMethod)) {
    errors.push(
      `accessMethod "${policy.accessMethod}" is not one of ${T.SOURCE_ACCESS_METHODS.join(', ')}`
    );
  }

  const base = hostOf(policy.baseUrl || policy.baseHost || '');
  const automated = policy.allowsAutomatedAccess === true;

  // Core rule (§6.2-6.4): a prohibited host may NOT allow automated access.
  if (automated && base && isProhibitedAutomationHost(base)) {
    errors.push(
      `base host "${base}" is a prohibited proprietary/LinkedIn/MLS source (§6.2-6.4); allowsAutomatedAccess must be false`
    );
  }

  // A prohibited host is only tolerable as manual_review_only / manual_upload.
  if (base && isProhibitedAutomationHost(base) && !NON_AUTOMATED_METHODS.includes(policy.accessMethod)) {
    errors.push(
      `base host "${base}" is prohibited for automation; accessMethod must be manual_review_only or manual_upload, got "${policy.accessMethod}"`
    );
  }

  // A web-automation access method combined with a prohibited base is illegal.
  const AUTOMATED_WEB_METHODS = ['first_party_public_web', 'rss_or_sitemap', 'official_bulk_download'];
  if (base && isProhibitedAutomationHost(base) && AUTOMATED_WEB_METHODS.includes(policy.accessMethod)) {
    errors.push(
      `access method "${policy.accessMethod}" performs web automation against prohibited host "${base}" (§6.1)`
    );
  }

  // Any prohibitedHosts entry that ALSO appears reachable is contradictory noise;
  // validate the declared prohibitedHosts are well-formed strings.
  if (policy.prohibitedHosts && !Array.isArray(policy.prohibitedHosts)) {
    errors.push('prohibitedHosts must be an array');
  }

  // If enabled, automated, and a web method, require conservative rate limits (§6.1, §23).
  if (policy.enabled && automated && AUTOMATED_WEB_METHODS.includes(policy.accessMethod)) {
    const rpm = policy.maxRequestsPerMinute;
    if (rpm != null && (typeof rpm !== 'number' || rpm <= 0 || rpm > 60)) {
      errors.push('maxRequestsPerMinute must be a sane 1-60 for automated web access (§23)');
    }
  }

  return { valid: errors.length === 0, errors };
}

/**
 * assertSourceEnabledLegal(policy) — throws if an ILLEGAL source is enabled for
 * automation. Called by the pipeline BEFORE any run (§10 "policy validation
 * before every run"). A disabled or non-automated policy is always allowed to
 * pass here (it does no fetching); only an ENABLED + AUTOMATED illegal source
 * is a hard stop.
 */
function assertSourceEnabledLegal(policy) {
  const { valid, errors } = validateSourcePolicy(policy);

  const base = hostOf((policy && (policy.baseUrl || policy.baseHost)) || '');
  const automated = policy && policy.allowsAutomatedAccess === true;
  const enabled = policy && policy.enabled === true;

  // The unforgivable combination: enabled + automated + prohibited host.
  if (enabled && automated && base && isProhibitedAutomationHost(base)) {
    throw new Error(
      `ILLEGAL SOURCE: "${(policy && policy.sourceKey) || base}" is enabled for automated access against prohibited host "${base}" (spec §6.2-6.4). Refusing to run.`
    );
  }

  // Any other structural invalidity on an enabled policy is also a stop.
  if (enabled && !valid) {
    throw new Error(
      `INVALID SOURCE POLICY "${(policy && policy.sourceKey) || '?'}": ${errors.join('; ')}`
    );
  }

  return true;
}

module.exports = {
  PROHIBITED_AUTOMATION_HOSTS,
  PROHIBITED_PROPRIETARY_HOSTS,
  validateSourcePolicy,
  assertSourceEnabledLegal,
  accessMethodAllowed,
  isProhibitedAutomationHost,
  looksLikeMls,
  hostOf,
};