`lib/federalRegister.ts`

185 lines
/**
 * lib/federalRegister.ts
 *
 * Real-data client for the free, open Federal Register API
 * (https://www.federalregister.gov/api/v1/documents.json — no API key required).
 *
 * Replaces the prior AI-fabricated "news" path that asked an LLM to invent
 * "realistic" headlines, "plausible" URLs, and "plausible" journalist names.
 * Every item this returns is a real, published Federal Register document with a
 * verifiable html_url on federalregister.gov.
 */

const FR_BASE = 'https://www.federalregister.gov/api/v1/documents.json';

/** Shape of a single agency entry inside an FR document. */
interface FRAgency {
  raw_name?: string;
  name?: string;
}

/** Subset of an FR document we request via fields[]. */
interface FRDocument {
  title?: string;
  abstract?: string;
  publication_date?: string;
  html_url?: string;
  agencies?: FRAgency[];
  type?: string;
  document_number?: string;
}

interface FRResponse {
  count?: number;
  results?: FRDocument[];
}

/** The normalized news item shape the rest of the app (DB + frontend) consumes. */
export interface NewsItem {
  headline: string;
  outlet: string;
  url: string | null;
  published_at: string | null;
  summary: string | null;
  tags: string[];
  relevance_score: number;
  author_name: string | null;
  source_type: string;
}

export type FRFetchResult =
  | { ok: true; items: NewsItem[]; count: number; term: string | null }
  | { ok: false; reason: 'network' | 'http' | 'parse'; detail: string };

/** FR document type → the app's source_type taxonomy (kept frontend-compatible). */
function mapSourceType(frType?: string): string {
  switch ((frType || '').toLowerCase()) {
    case 'rule':
    case 'proposed rule':
      return 'policy_brief';
    case 'notice':
      return 'press_release';
    case 'presidential document':
      return 'industry';
    default:
      return 'google_news';
  }
}

/** Best human-readable agency name for the "outlet" slot. */
function primaryAgency(agencies?: FRAgency[]): string {
  if (!agencies || agencies.length === 0) return 'Federal Register';
  const a = agencies[0];
  return a.name || a.raw_name || 'Federal Register';
}

/**
 * Lightweight relevance score in [0,1]. The FR term-search already filters to
 * matching docs, so a real (non-fabricated) heuristic just nudges items whose
 * title/abstract actually contain the org's keywords higher. Defaults to 0.5
 * when no keywords are supplied.
 */
function scoreRelevance(doc: FRDocument, keywords: string[]): number {
  if (keywords.length === 0) return 0.5;
  const hay = `${doc.title || ''} ${doc.abstract || ''}`.toLowerCase();
  let hits = 0;
  for (const kw of keywords) {
    const k = kw.trim().toLowerCase();
    if (k && hay.includes(k)) hits++;
  }
  if (hits === 0) return 0.5;
  // 1 hit → 0.7, scaling up toward 0.95 with more keyword matches.
  return Math.min(0.95, 0.6 + 0.1 * hits + 0.05);
}

/** Build the tag list from the FR doc type + matched org keywords. */
function buildTags(doc: FRDocument, keywords: string[]): string[] {
  const tags = new Set<string>();
  if (doc.type) tags.add(doc.type);
  tags.add('Federal Register');
  const hay = `${doc.title || ''} ${doc.abstract || ''}`.toLowerCase();
  for (const kw of keywords) {
    const k = kw.trim();
    if (k && hay.includes(k.toLowerCase())) tags.add(k);
  }
  return Array.from(tags).slice(0, 5);
}

/**
 * Fetch real, recent Federal Register documents and map them into the app's
 * news-item shape.
 *
 * @param opts.keywords   org keywords; the first few drive an FR term search
 *                        and also feed relevance scoring + tagging.
 * @param opts.perPage    how many docs to request (1-100). Default 20.
 * @param opts.timeoutMs  network timeout. Default 15000.
 */
export async function fetchFederalRegisterNews(opts: {
  keywords?: string[];
  perPage?: number;
  timeoutMs?: number;
} = {}): Promise<FRFetchResult> {
  const keywords = (opts.keywords || []).filter((k) => typeof k === 'string' && k.trim());
  const perPage = Math.min(Math.max(opts.perPage ?? 20, 1), 100);
  const timeoutMs = opts.timeoutMs ?? 15000;

  // Use up to 4 keywords joined as a single OR-ish term search. The FR API
  // treats `conditions[term]` as a relevance full-text query.
  const term = keywords.length > 0 ? keywords.slice(0, 4).join(' ') : null;

  const params = new URLSearchParams();
  params.append('conditions[type][]', 'RULE');
  params.append('conditions[type][]', 'NOTICE');
  params.append('conditions[type][]', 'PRORULE');
  if (term) params.append('conditions[term]', term);
  params.append('per_page', String(perPage));
  params.append('order', term ? 'relevance' : 'newest');
  for (const f of ['title', 'abstract', 'publication_date', 'html_url', 'agencies', 'type', 'document_number']) {
    params.append('fields[]', f);
  }

  const url = `${FR_BASE}?${params.toString()}`;

  let res: Response;
  const controller = new AbortController();
  const timer = setTimeout(() => controller.abort(), timeoutMs);
  try {
    res = await fetch(url, {
      signal: controller.signal,
      headers: { Accept: 'application/json', 'User-Agent': 'Grant-Nonprofit-NewsMonitor/1.0' },
    });
  } catch (err) {
    clearTimeout(timer);
    return { ok: false, reason: 'network', detail: (err as Error).message };
  }
  clearTimeout(timer);

  if (!res.ok) {
    return { ok: false, reason: 'http', detail: `Federal Register API returned ${res.status}` };
  }

  let body: FRResponse;
  try {
    body = (await res.json()) as FRResponse;
  } catch (err) {
    return { ok: false, reason: 'parse', detail: (err as Error).message };
  }

  const results = Array.isArray(body.results) ? body.results : [];
  const items: NewsItem[] = results
    .filter((d) => d.title && d.html_url)
    .map((d) => ({
      headline: d.title!.trim(),
      outlet: primaryAgency(d.agencies),
      url: d.html_url || null,
      published_at: d.publication_date || null,
      summary: d.abstract?.trim() || null,
      tags: buildTags(d, keywords),
      relevance_score: scoreRelevance(d, keywords),
      author_name: 'Federal Register', // source attribution; FR docs have no byline
      source_type: mapSourceType(d.type),
    }));

  return { ok: true, items, count: body.count ?? items.length, term };
}