← back to Norma

app/api/ingest/site/route.ts

197 lines

import { NextRequest, NextResponse } from 'next/server';
import { requireRole } from '@/lib/require-role';
import { auditLog } from '@/lib/audit';
import {
  crawlOrgSite,
  crawlOrgNews,
  crawlOrgAbout,
  crawlOrgResources,
  type CrawlOptions,
  type CrawlResult,
  type CrawlSummary,
} from '@/lib/ingest-site';

/**
 * POST /api/ingest/site
 *
 * Trigger a crawl of the org website to populate the site_documents
 * table with voice/style corpus for email generation.
 *
 * Query params:
 *   ?section=news       — crawl only the news section (RSS feed + post pages)
 *   ?section=about      — crawl only the /about page
 *   ?section=resources  — crawl only the /resources page
 *   (omit section)      — crawl all three sections sequentially
 *
 * Optional JSON body:
 *   {
 *     "maxPostDetail": 10,   // max news posts to fetch full-page detail for (default 20)
 *     "timeoutMs": 20000     // per-page fetch timeout in ms (default 15000)
 *   }
 *
 * Response 200:
 *   Full crawl:
 *   {
 *     "results": [
 *       { "section": "news",      "fetched": 20, "inserted": 15, "updated": 5,  "errors": 0 },
 *       { "section": "about",     "fetched": 1,  "inserted": 0,  "updated": 1,  "errors": 0 },
 *       { "section": "resources", "fetched": 1,  "inserted": 0,  "updated": 1,  "errors": 0 }
 *     ],
 *     "totalFetched": 22,
 *     "totalInserted": 15,
 *     "totalUpdated": 7,
 *     "totalErrors": 0,
 *     "durationMs": 38412
 *   }
 *
 *   Single-section crawl:
 *   {
 *     "result": { "section": "news", "fetched": 20, "inserted": 15, "updated": 5, "errors": 0 },
 *     "durationMs": 31200
 *   }
 */
export async function POST(request: NextRequest) {
  const auth = requireRole(request, 'admin', 'staff');
  if (auth instanceof NextResponse) return auth;

  const startMs = Date.now();
  const ip =
    request.headers.get('x-forwarded-for') ||
    request.headers.get('x-real-ip') ||
    undefined;

  // Parse optional JSON body — gracefully ignore empty or non-JSON bodies
  let bodyOptions: CrawlOptions = {};
  try {
    const text = await request.text();
    if (text.trim()) {
      const parsed = JSON.parse(text);
      bodyOptions = {
        maxPostDetail:
          typeof parsed.maxPostDetail === 'number' ? parsed.maxPostDetail : undefined,
        timeoutMs:
          typeof parsed.timeoutMs === 'number' ? parsed.timeoutMs : undefined,
      };
    }
  } catch {
    // Non-JSON body is fine — proceed with defaults
  }

  // Determine which section to crawl (or all sections if not specified)
  const { searchParams } = new URL(request.url);
  const sectionParam = searchParams.get('section');

  const validSections = ['news', 'about', 'resources'];
  if (sectionParam !== null && !validSections.includes(sectionParam)) {
    return NextResponse.json(
      {
        error: `Invalid section parameter "${sectionParam}". Must be one of: ${validSections.join(', ')}.`,
      },
      { status: 400 },
    );
  }

  // Run the appropriate crawl function
  if (sectionParam === null) {
    // Full site crawl — all three sections sequentially
    let summary: CrawlSummary;

    try {
      summary = await crawlOrgSite(bodyOptions);
    } catch (err) {
      const message = (err as Error).message;
      console.error('[api/ingest/site] crawlOrgSite threw:', message);

      await auditLog(
        'ingest.site.failed',
        'site_documents',
        null,
        { section: 'all', error: message },
        ip,
      );

      return NextResponse.json({ error: `Site crawl failed: ${message}` }, { status: 500 });
    }

    const durationMs = Date.now() - startMs;

    await auditLog(
      'ingest.site.triggered',
      'site_documents',
      null,
      {
        section: 'all',
        totalFetched: summary.totalFetched,
        totalInserted: summary.totalInserted,
        totalUpdated: summary.totalUpdated,
        totalErrors: summary.totalErrors,
        durationMs,
        options: bodyOptions,
      },
      ip,
    );

    return NextResponse.json(
      {
        results: summary.results,
        totalFetched: summary.totalFetched,
        totalInserted: summary.totalInserted,
        totalUpdated: summary.totalUpdated,
        totalErrors: summary.totalErrors,
        durationMs,
      },
      { status: 200 },
    );
  }

  // Single-section crawl
  const sectionCrawlers: Record<string, (opts: CrawlOptions) => Promise<CrawlResult>> = {
    news: crawlOrgNews,
    about: crawlOrgAbout,
    resources: crawlOrgResources,
  };

  const crawlFn = sectionCrawlers[sectionParam];
  let result: CrawlResult;

  try {
    result = await crawlFn(bodyOptions);
  } catch (err) {
    const message = (err as Error).message;
    console.error(`[api/ingest/site] crawl(${sectionParam}) threw:`, message);

    await auditLog(
      'ingest.site.failed',
      'site_documents',
      null,
      { section: sectionParam, error: message },
      ip,
    );

    return NextResponse.json(
      { error: `Crawl of section "${sectionParam}" failed: ${message}` },
      { status: 500 },
    );
  }

  const durationMs = Date.now() - startMs;

  await auditLog(
    'ingest.site.triggered',
    'site_documents',
    null,
    {
      section: sectionParam,
      fetched: result.fetched,
      inserted: result.inserted,
      updated: result.updated,
      errors: result.errors,
      durationMs,
      options: bodyOptions,
    },
    ip,
  );

  return NextResponse.json({ result, durationMs }, { status: 200 });
}