← back to Norma

app/api/email-analyzer/fetch-url/route.ts

281 lines

/**
 * POST /api/email-analyzer/fetch-url
 * Fetches a URL server-side, extracts article content, and uses Gemini to extract key points.
 */

import { NextRequest, NextResponse } from 'next/server';
import { requireRole } from '@/lib/require-role';
import { checkRateLimit } from '@/lib/rate-limit';
import { assertPublicUrl } from '@/lib/ssrf-guard';

const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
const GEMINI_URL =
  'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent';

/* ─── HTML parsing helpers (regex-based, no external deps) ──────────────── */

function stripTags(html: string): string {
  return html
    .replace(/<script[\s\S]*?<\/script>/gi, '')
    .replace(/<style[\s\S]*?<\/style>/gi, '')
    .replace(/<noscript[\s\S]*?<\/noscript>/gi, '')
    .replace(/<[^>]+>/g, ' ')
    .replace(/&nbsp;/gi, ' ')
    .replace(/&amp;/gi, '&')
    .replace(/&lt;/gi, '<')
    .replace(/&gt;/gi, '>')
    .replace(/&quot;/gi, '"')
    .replace(/&#39;/gi, "'")
    .replace(/\s+/g, ' ')
    .trim();
}

function extractMeta(html: string, name: string): string | null {
  // Try name="..." content="..."
  const nameRe = new RegExp(
    `<meta[^>]+(?:name|property)=["']${name}["'][^>]+content=["']([^"']+)["']`,
    'i',
  );
  const match = html.match(nameRe);
  if (match) return match[1].trim();

  // Try content="..." name="..." (reversed order)
  const revRe = new RegExp(
    `<meta[^>]+content=["']([^"']+)["'][^>]+(?:name|property)=["']${name}["']`,
    'i',
  );
  const revMatch = html.match(revRe);
  if (revMatch) return revMatch[1].trim();

  return null;
}

function extractTitle(html: string): string {
  // Try og:title first
  const ogTitle = extractMeta(html, 'og:title');
  if (ogTitle) return ogTitle;

  // Fallback to <title>
  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
  if (titleMatch) return titleMatch[1].trim();

  return 'Untitled Page';
}

function extractDescription(html: string): string | null {
  return (
    extractMeta(html, 'og:description') ||
    extractMeta(html, 'description') ||
    extractMeta(html, 'twitter:description') ||
    null
  );
}

function extractAuthor(html: string): string | null {
  return extractMeta(html, 'author') || extractMeta(html, 'article:author') || null;
}

function extractPublishedDate(html: string): string | null {
  return (
    extractMeta(html, 'article:published_time') ||
    extractMeta(html, 'datePublished') ||
    extractMeta(html, 'date') ||
    null
  );
}

function extractArticleContent(html: string): string {
  // Try to find the main article content in priority order
  const contentPatterns = [
    /<article[^>]*>([\s\S]*?)<\/article>/i,
    /<main[^>]*>([\s\S]*?)<\/main>/i,
    /<div[^>]*class=["'][^"']*(?:article|post|content|entry|story)[^"']*["'][^>]*>([\s\S]*?)<\/div>/i,
    /<div[^>]*role=["']main["'][^>]*>([\s\S]*?)<\/div>/i,
  ];

  for (const pattern of contentPatterns) {
    const match = html.match(pattern);
    if (match) {
      const text = stripTags(match[1]);
      if (text.length > 100) return text;
    }
  }

  // Fallback: extract all <p> tags and join them
  const paragraphs: string[] = [];
  const pRegex = /<p[^>]*>([\s\S]*?)<\/p>/gi;
  let pMatch;
  while ((pMatch = pRegex.exec(html)) !== null) {
    const text = stripTags(pMatch[1]).trim();
    if (text.length > 20) {
      paragraphs.push(text);
    }
  }

  if (paragraphs.length > 0) {
    return paragraphs.join(' ');
  }

  // Last resort: strip all tags from body
  const bodyMatch = html.match(/<body[^>]*>([\s\S]*?)<\/body>/i);
  if (bodyMatch) {
    return stripTags(bodyMatch[1]);
  }

  return stripTags(html);
}

function extractDomain(url: string): string {
  try {
    return new URL(url).hostname.replace(/^www\./, '');
  } catch {
    return 'unknown';
  }
}

/* ─── Gemini key points extraction ──────────────────────────────────────── */

async function extractKeyPoints(
  content: string,
  title: string,
): Promise<string[]> {
  if (!GEMINI_API_KEY || !content || content.length < 50) return [];

  const truncated = content.substring(0, 3000);
  const prompt = `Extract 3-5 key points from this article. Return ONLY a JSON array of strings (each point is 1-2 sentences). No explanation, no markdown — just the JSON array.

Title: ${title}

Content:
${truncated}`;

  try {
    const res = await fetch(`${GEMINI_URL}?key=${GEMINI_API_KEY}`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        contents: [{ parts: [{ text: prompt }] }],
        generationConfig: { temperature: 0.3, maxOutputTokens: 500 },
      }),
      signal: AbortSignal.timeout(15_000),
    });

    if (!res.ok) return [];

    const data = await res.json();
    let text = data?.candidates?.[0]?.content?.parts?.[0]?.text || '';
    // Strip code fences if present
    text = text.replace(/^```json?\s*\n?/i, '').replace(/\n?```\s*$/i, '').trim();

    const parsed = JSON.parse(text);
    if (Array.isArray(parsed)) {
      return parsed.filter((p: unknown) => typeof p === 'string').slice(0, 5);
    }
    return [];
  } catch {
    return [];
  }
}

/* ─── Route handler ─────────────────────────────────────────────────────── */

export async function POST(request: NextRequest) {
  const auth = requireRole(request, 'admin', 'staff');
  if (auth instanceof NextResponse) return auth;

  // Rate limit: 5 per minute
  const rl = checkRateLimit(request, 5, 60_000, 'fetch-url');
  if (rl.limited) {
    return NextResponse.json(
      { error: 'Too many URL fetch requests. Try again in a minute.' },
      { status: 429 },
    );
  }

  try {
    const body = await request.json();
    const { url } = body;

    if (!url || typeof url !== 'string') {
      return NextResponse.json({ error: 'url is required' }, { status: 400 });
    }

    // SSRF guard: validate protocol + block private/reserved/loopback hosts
    // (also resolves DNS so a hostname pointing at a private IP is blocked).
    const guard = await assertPublicUrl(url);
    if (!guard.ok) {
      return NextResponse.json({ error: guard.reason }, { status: 400 });
    }
    const safeUrl = guard.url!;

    // Fetch the page with a 10-second timeout
    let html: string;
    try {
      const res = await fetch(safeUrl, {
        headers: {
          'User-Agent':
            'Mozilla/5.0 (compatible; NormaBot/1.0; +https://norma.app)',
          Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
          'Accept-Language': 'en-US,en;q=0.5',
        },
        redirect: 'follow',
        signal: AbortSignal.timeout(10_000),
      });

      if (!res.ok) {
        return NextResponse.json(
          { error: `Failed to fetch URL (HTTP ${res.status})` },
          { status: 422 },
        );
      }

      const contentType = res.headers.get('content-type') || '';
      if (!contentType.includes('text/html') && !contentType.includes('application/xhtml')) {
        return NextResponse.json(
          { error: 'URL does not appear to be an HTML page' },
          { status: 422 },
        );
      }

      html = await res.text();
    } catch (fetchErr) {
      const message = (fetchErr as Error).message || 'Unknown error';
      if (message.includes('abort') || message.includes('timeout')) {
        return NextResponse.json(
          { error: 'URL fetch timed out (10 second limit)' },
          { status: 408 },
        );
      }
      return NextResponse.json(
        { error: `Failed to fetch URL: ${message}` },
        { status: 422 },
      );
    }

    // Extract metadata and content
    const title = extractTitle(html);
    const description = extractDescription(html);
    const author = extractAuthor(html);
    const publishedDate = extractPublishedDate(html);
    const rawContent = extractArticleContent(html);
    const content = rawContent.substring(0, 2000);
    const source = extractDomain(safeUrl);

    // Extract key points via Gemini
    const keyPoints = await extractKeyPoints(rawContent, title);

    return NextResponse.json({
      title,
      description,
      content,
      source,
      author,
      publishedDate,
      keyPoints,
    });
  } catch (err) {
    console.error('[email-analyzer/fetch-url] error:', (err as Error).message);
    return NextResponse.json({ error: 'Failed to process URL' }, { status: 500 });
  }
}