`app/api/news/live/route.ts`

297 lines
import { NextRequest, NextResponse } from 'next/server';
import { query } from '@/lib/db';
import { requireRole } from '@/lib/require-role';
import { getOrgId } from '@/lib/orgId';

// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------

interface NewsRow {
  id: string;
  headline: string;
  outlet: string | null;
  url: string | null;
  published_at: string | null;
  summary: string | null;
  tags: string[] | null;
  relevance_score: number | null;
  is_used: boolean;
  session_id: string | null;
  source_type: string;
  raw_data: Record<string, unknown> | null;
  created_at: string;
  updated_at: string;
  author_name: string | null;
  author_linkedin: string | null;
}

interface RssArticle {
  title: string;
  link: string;
  pubDate: string;
  source: string;
}

interface LiveNewsResponse {
  articles: NewsRow[];
  freshCount: number;
  scrapedCount: number;
  lastScraped: string;
}

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

const RSS_FEEDS = [
  'https://news.google.com/rss/search?q=student+loan+debt+forgiveness&hl=en-US&gl=US&ceid=US:en',
  'https://news.google.com/rss/search?q=SAVE+plan+student+loan+repayment&hl=en-US&gl=US&ceid=US:en',
  'https://news.google.com/rss/search?q=CFPB+student+loan+servicer&hl=en-US&gl=US&ceid=US:en',
];

const FRESHNESS_INTERVAL = '2 hours';
const FRESH_THRESHOLD = 10;
const FETCH_TIMEOUT_MS = 10_000;
const MAX_ARTICLES = 50;

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/**
 * Build a tags array based on keyword matches in the headline.
 * Always includes 'student-debt' as the base tag.
 */
function buildTags(title: string): string[] {
  const tags: string[] = ['student-debt'];
  const lower = title.toLowerCase();

  if (lower.includes('forgiv')) tags.push('forgiveness');
  if (lower.includes('save')) tags.push('SAVE-plan');
  if (lower.includes('repay')) tags.push('repayment');
  if (lower.includes('cfpb') || lower.includes('consumer')) tags.push('CFPB');
  if (lower.includes('servicer') || lower.includes('mohela') || lower.includes('navient')) tags.push('servicer');
  if (lower.includes('default') || lower.includes('collection')) tags.push('collections');
  if (lower.includes('biden') || lower.includes('trump')) tags.push('policy');

  return tags;
}

/**
 * Parse RSS XML and extract article items using regex.
 * Returns an array of RssArticle objects.
 */
function parseRssXml(xml: string): RssArticle[] {
  const articles: RssArticle[] = [];
  const items = xml.match(/<item>([\s\S]*?)<\/item>/g) || [];

  for (const item of items) {
    const title =
      item
        .match(/<title>(.*?)<\/title>/)?.[1]
        ?.replace(/<!\[CDATA\[|\]\]>/g, '')
        .trim() || '';
    const link = item.match(/<link\/?>(\s*)([\s\S]*?)(?=<)/)?.[2]?.trim() ||
      item.match(/<link>(.*?)<\/link>/)?.[1]?.trim() || '';
    const pubDate = item.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() || '';
    const source =
      item
        .match(/<source.*?>(.*?)<\/source>/)?.[1]
        ?.replace(/<!\[CDATA\[|\]\]>/g, '')
        .trim() || '';

    if (title && link) {
      articles.push({ title, link, pubDate, source });
    }
  }

  return articles;
}

/**
 * Fetch a single RSS feed with timeout. Returns parsed articles or empty array on error.
 */
async function fetchRssFeed(url: string): Promise<RssArticle[]> {
  try {
    const response = await fetch(url, {
      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
      headers: {
        'User-Agent': 'Norma-NewsBot/1.0',
        Accept: 'application/rss+xml, application/xml, text/xml',
      },
    });

    if (!response.ok) {
      console.error(`[api/news/live] RSS fetch failed (${response.status}): ${url}`);
      return [];
    }

    const xml = await response.text();
    return parseRssXml(xml);
  } catch (err) {
    console.error(`[api/news/live] RSS fetch error for ${url}:`, (err as Error).message);
    return [];
  }
}

/**
 * Check whether a headline already exists in news_items (case-insensitive).
 */
async function headlineExists(headline: string, orgId: string | null): Promise<boolean> {
  if (orgId) {
    const result = await query(
      'SELECT 1 FROM news_items WHERE LOWER(headline) = LOWER($1) AND org_id = $2 LIMIT 1',
      [headline, orgId],
    );
    return result.rows.length > 0;
  }
  const result = await query(
    'SELECT 1 FROM news_items WHERE LOWER(headline) = LOWER($1) LIMIT 1',
    [headline],
  );
  return result.rows.length > 0;
}

/**
 * Insert a single scraped article into the database.
 * Returns the inserted row, or null if it was a duplicate / insert failed.
 */
async function insertArticle(article: RssArticle, orgId: string | null): Promise<NewsRow | null> {
  const tags = buildTags(article.title);

  // Parse published date; fall back to now() if unparseable
  let publishedAt: Date | null = null;
  if (article.pubDate) {
    const parsed = new Date(article.pubDate);
    if (!isNaN(parsed.getTime())) {
      publishedAt = parsed;
    }
  }

  try {
    const result = await query<NewsRow>(
      `INSERT INTO news_items (headline, outlet, url, published_at, summary, tags, relevance_score, source_type, org_id)
       VALUES ($1, $2, $3, $4, $5, $6, $7, 'live_scrape', $8)
       ON CONFLICT DO NOTHING
       RETURNING *`,
      [
        article.title,
        article.source || null,
        article.link || null,
        publishedAt ? publishedAt.toISOString() : null,
        null, // summary — not available from RSS
        tags,
        null, // relevance_score
        orgId,
      ],
    );

    return result.rows[0] ?? null;
  } catch (err) {
    console.error('[api/news/live] Insert error:', (err as Error).message);
    return null;
  }
}

// ---------------------------------------------------------------------------
// GET /api/news/live
// ---------------------------------------------------------------------------

export async function GET(request: NextRequest) {
  const auth = requireRole(request, 'admin', 'staff');
  if (auth instanceof NextResponse) return auth;

  const orgId = auth.role === 'admin' ? getOrgId(request) : auth.orgId;

  try {
    // 1. Check for fresh articles in the database
    const orgFilter = orgId ? ' AND org_id = $1' : '';
    const orgParams = orgId ? [orgId] : [];

    const freshResult = await query<NewsRow>(
      `SELECT * FROM news_items
       WHERE created_at > NOW() - INTERVAL '${FRESHNESS_INTERVAL}'${orgFilter}
       ORDER BY published_at DESC NULLS LAST
       LIMIT ${MAX_ARTICLES}`,
      orgParams,
    );

    const freshCount = freshResult.rows.length;
    let scrapedCount = 0;
    const lastScrapedTime = new Date().toISOString();

    // 2. If we have fewer than the threshold, scrape new articles
    if (freshCount < FRESH_THRESHOLD) {
      // Fetch all RSS feeds in parallel; each feed handles its own errors
      const feedResults = await Promise.all(RSS_FEEDS.map(fetchRssFeed));
      const allArticles: RssArticle[] = feedResults.flat();

      // Deduplicate within the scraped batch by title (case-insensitive)
      const seenTitles = new Set<string>();
      const uniqueArticles: RssArticle[] = [];
      for (const article of allArticles) {
        const key = article.title.toLowerCase();
        if (!seenTitles.has(key)) {
          seenTitles.add(key);
          uniqueArticles.push(article);
        }
      }

      // Insert each article, skipping duplicates against the DB
      for (const article of uniqueArticles) {
        const exists = await headlineExists(article.title, orgId);
        if (exists) continue;

        const inserted = await insertArticle(article, orgId);
        if (inserted) {
          scrapedCount++;
        }
      }
    }

    // 3. Return all articles (fresh + newly scraped), sorted by published_at DESC
    const finalOrgFilter = orgId ? ' WHERE org_id = $1' : '';
    const finalResult = await query<NewsRow>(
      `SELECT * FROM news_items${finalOrgFilter}
       ORDER BY published_at DESC NULLS LAST
       LIMIT ${MAX_ARTICLES}`,
      orgParams,
    );

    const response: LiveNewsResponse = {
      articles: finalResult.rows,
      freshCount,
      scrapedCount,
      lastScraped: lastScrapedTime,
    };

    return NextResponse.json(response);
  } catch (err) {
    console.error('[api/news/live] GET error:', (err as Error).message);

    // Even on error, attempt to return whatever is in the DB
    try {
      const fallbackOrgFilter = orgId ? ' WHERE org_id = $1' : '';
      const fallbackParams = orgId ? [orgId] : [];
      const fallbackResult = await query<NewsRow>(
        `SELECT * FROM news_items${fallbackOrgFilter}
         ORDER BY published_at DESC NULLS LAST
         LIMIT ${MAX_ARTICLES}`,
        fallbackParams,
      );

      return NextResponse.json({
        articles: fallbackResult.rows,
        freshCount: fallbackResult.rows.length,
        scrapedCount: 0,
        lastScraped: new Date().toISOString(),
        error: 'Partial failure — returning cached articles',
      });
    } catch (dbErr) {
      console.error('[api/news/live] Fallback DB query also failed:', (dbErr as Error).message);
      return NextResponse.json({ error: 'Failed to fetch live news' }, { status: 500 });
    }
  }
}