`app/api/ingest/regulations/route.ts`

124 lines
import { NextRequest, NextResponse } from 'next/server';
import { query } from '@/lib/db';
import { requireRole } from '@/lib/require-role';
import { getOrgId } from '@/lib/orgId';
import { searchDocuments, type Regulation } from '@/lib/regulations-gov';

/**
 * POST /api/ingest/regulations
 *
 * Ingests federal regulations from Regulations.gov API.
 * Body: { keywords?: string[], agencyId?: string }
 * Upserts into federal_regulations table.
 */
export async function POST(request: NextRequest) {
  const auth = requireRole(request, 'admin', 'staff');
  if (auth instanceof NextResponse) return auth;

  const orgId = auth.role === 'admin' ? getOrgId(request) : auth.orgId;

  let body: { keywords?: string[]; agencyId?: string } = {};
  try {
    body = await request.json();
  } catch {
    /* use defaults */
  }

  const keywords = body.keywords ?? ['education', 'nonprofit', 'public health'];
  const agencyId = body.agencyId ?? undefined;

  const stats = { ingested: 0, errors: 0 };
  const errorDetails: string[] = [];

  // Deduplicate across keyword searches
  const allDocs = new Map<string, Regulation>();

  for (const keyword of keywords) {
    try {
      console.log('[ingest/regulations] Searching documents for:', keyword);
      const docs = await searchDocuments(keyword, agencyId, undefined, 50);
      for (const doc of docs) {
        if (doc.documentId && !allDocs.has(doc.documentId)) {
          allDocs.set(doc.documentId, doc);
        }
      }
    } catch (err) {
      console.error('[ingest/regulations] Search error for keyword:', keyword, (err as Error).message);
      errorDetails.push('search_' + keyword + ': ' + (err as Error).message);
      stats.errors++;
    }
  }

  console.log('[ingest/regulations] Found', allDocs.size, 'unique documents to upsert');

  for (const doc of allDocs.values()) {
    try {
      await query(
        `INSERT INTO federal_regulations (
          document_id, title, agency_id, document_type,
          posted_date, comment_end_date, docket_id, url,
          raw_json, org_id
        ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
        ON CONFLICT (document_id) DO UPDATE SET
          title = EXCLUDED.title,
          agency_id = EXCLUDED.agency_id,
          document_type = EXCLUDED.document_type,
          posted_date = EXCLUDED.posted_date,
          comment_end_date = EXCLUDED.comment_end_date,
          docket_id = EXCLUDED.docket_id,
          url = EXCLUDED.url,
          raw_json = EXCLUDED.raw_json,
          updated_at = NOW()`,
        [
          doc.documentId,
          doc.title,
          doc.agencyId || null,
          doc.documentType || null,
          doc.postedDate || null,
          doc.commentEndDate || null,
          doc.docketId || null,
          doc.url || null,
          JSON.stringify(doc),
          orgId || null,
        ],
      );
      stats.ingested++;
    } catch (err) {
      console.error('[ingest/regulations] Insert error:', (err as Error).message);
      errorDetails.push('insert: ' + (err as Error).message);
      stats.errors++;
    }
  }

  console.log('[ingest/regulations] Done:', stats.ingested, 'ingested,', stats.errors, 'errors');

  return NextResponse.json({
    success: true,
    ingested: stats.ingested,
    errors: stats.errors,
    ...(errorDetails.length > 0 ? { errorDetails } : {}),
  });
}

/* ─── GET: Return ingested federal regulations ───────────────────────────── */
export async function GET(request: NextRequest) {
  const auth = requireRole(request, 'admin', 'staff');
  if (auth instanceof NextResponse) return auth;

  try {
    const { rows } = await query(`
      SELECT id, document_id, title, agency_id, agency_name,
             document_type, posted_date, comment_end_date,
             docket_id, summary, url, federal_register_number, created_at
      FROM federal_regulations
      ORDER BY posted_date DESC NULLS LAST, created_at DESC
      LIMIT 200
    `);

    return NextResponse.json({ regulations: rows, total: rows.length });
  } catch (err) {
    console.error('[ingest/regulations] GET error:', (err as Error).message);
    return NextResponse.json({ error: 'Failed to fetch federal regulations' }, { status: 500 });
  }
}