← back to Norma
app/api/journalists/scrape/route.ts
360 lines
import { NextRequest, NextResponse } from 'next/server';
import { query } from '@/lib/db';
import { requireRole } from '@/lib/require-role';
import { createJournalistLinks } from '@/lib/journalist-links';
const GEMINI_KEY = process.env.GEMINI_API_KEY || '';
const GEMINI_URL = `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GEMINI_KEY}`;
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
// ─── RSS helpers (adapted from lib/ingest-news.ts) ─────────────────────────
function extractTag(xml: string, tag: string): string | null {
const re = new RegExp(`<${tag}[^>]*>(?:<!\\[CDATA\\[)?([\\s\\S]*?)(?:\\]\\]>)?</${tag}>`, 'i');
const match = xml.match(re);
return match ? match[1].trim() || null : null;
}
function cleanTitle(raw: string): string {
return raw.replace(/<[^>]+>/g, '').replace(/&/g, '&').replace(/</g, '<')
.replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'").trim();
}
function deriveSourceName(itemXml: string): string {
const sourceMatch = itemXml.match(/<source[^>]*>([^<]+)<\/source>/i);
if (sourceMatch) return sourceMatch[1].trim();
const titleRaw = extractTag(itemXml, 'title') || '';
const dashParts = titleRaw.split(' - ');
if (dashParts.length > 1) return dashParts[dashParts.length - 1].trim();
return 'Unknown';
}
// ─── HTML helpers ──────────────────────────────────────────────────────────
async function resolveGoogleNewsUrl(gnUrl: string): Promise<string> {
if (!gnUrl.includes('news.google.com')) return gnUrl;
try {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), 6000);
const res = await fetch(gnUrl, { signal: ctrl.signal, redirect: 'manual', headers: { 'User-Agent': UA } });
clearTimeout(timer);
const loc = res.headers.get('location');
if (loc && !loc.includes('news.google.com')) return loc;
const html = await res.text();
const jsRedirect = html.match(/window\.location\.replace\(['"]([^'"]+)['"]\)/);
if (jsRedirect) return jsRedirect[1];
const metaRefresh = html.match(/content="\d+;\s*url=([^"]+)"/i);
if (metaRefresh) return metaRefresh[1];
return gnUrl;
} catch {
return gnUrl;
}
}
function extractAuthorFromHtml(html: string): string | null {
// JSON-LD
const jsonLdMatch = html.match(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi);
if (jsonLdMatch) {
for (const block of jsonLdMatch) {
const jsonStr = block.replace(/<\/?script[^>]*>/gi, '');
try {
const data = JSON.parse(jsonStr);
const items = Array.isArray(data) ? data : [data];
for (const item of items) {
if (item?.author) {
const author = Array.isArray(item.author) ? item.author[0] : item.author;
const name = typeof author === 'string' ? author : author?.name;
if (name && name.length > 2 && name.length < 80) return name.trim();
}
}
} catch { /* ignore parse errors */ }
}
}
// Meta tags
const metaPatterns = [
/name="author"\s+content="([^"]+)"/i,
/property="article:author"\s+content="([^"]+)"/i,
/name="parsely-author"\s+content="([^"]+)"/i,
];
for (const pat of metaPatterns) {
const m = html.match(pat);
if (m && m[1].length > 2 && m[1].length < 80) return m[1].trim();
}
return null;
}
function extractArticleText(html: string): string {
// Remove scripts, styles, nav, header, footer
let text = html
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<nav[\s\S]*?<\/nav>/gi, '')
.replace(/<header[\s\S]*?<\/header>/gi, '')
.replace(/<footer[\s\S]*?<\/footer>/gi, '')
.replace(/<[^>]+>/g, ' ')
.replace(/ /g, ' ')
.replace(/&/g, '&')
.replace(/\s+/g, ' ')
.trim();
return text.slice(0, 5000);
}
function nameMatches(extracted: string, searched: string): boolean {
const a = extracted.toLowerCase().trim();
const b = searched.toLowerCase().trim();
if (a === b) return true;
if (a.includes(b) || b.includes(a)) return true;
// Check last name match
const aParts = a.split(/\s+/);
const bParts = b.split(/\s+/);
if (aParts.length >= 2 && bParts.length >= 2) {
return aParts[aParts.length - 1] === bParts[bParts.length - 1]
&& aParts[0][0] === bParts[0][0];
}
return false;
}
// ─── Gemini NLP ────────────────────────────────────────────────────────────
interface ArticleAnalysis {
summary: string;
topics: string[];
sentiment: string;
mentioned_people: string[];
mentioned_orgs: string[];
tags: string[];
}
async function analyzeArticle(title: string, outlet: string, content: string): Promise<ArticleAnalysis | null> {
try {
const prompt = `Analyze this news article. Return ONLY valid JSON (no markdown, no code fences):
{
"summary": "1-2 sentence factual summary",
"topics": ["3-5 topic keywords relevant to the article's subject area"],
"sentiment": "positive|negative|neutral|mixed",
"mentioned_people": ["full names of people mentioned or quoted"],
"mentioned_orgs": ["organization/company names mentioned"],
"tags": ["2-4 categorization tags"]
}
Article title: "${title}"
Article outlet: "${outlet}"
Article text (first 3000 chars):
${content.slice(0, 3000)}`;
const res = await fetch(GEMINI_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
contents: [{ parts: [{ text: prompt }] }],
generationConfig: { temperature: 0.3, maxOutputTokens: 1000 },
}),
});
if (!res.ok) return null;
const data = await res.json();
let text = data?.candidates?.[0]?.content?.parts?.[0]?.text ?? '';
// Strip code fences
text = text.replace(/```json\s*/gi, '').replace(/```\s*/gi, '').trim();
return JSON.parse(text) as ArticleAnalysis;
} catch {
return null;
}
}
// ─── POST /api/journalists/scrape ──────────────────────────────────────────
export async function POST(request: NextRequest) {
const auth = requireRole(request, 'admin', 'staff');
if (auth instanceof NextResponse) return auth;
try {
const { name, outlet } = await request.json();
if (!name || typeof name !== 'string' || name.trim().length < 2) {
return NextResponse.json({ error: 'Reporter name required (min 2 chars)' }, { status: 400 });
}
const reporterName = name.trim();
// 1. Search Google News RSS
const rssUrl = `https://news.google.com/rss/search?q=%22${encodeURIComponent(reporterName)}%22&hl=en-US&gl=US&ceid=US:en`;
const rssRes = await fetch(rssUrl, { headers: { 'User-Agent': UA } });
if (!rssRes.ok) {
return NextResponse.json({ error: 'Failed to fetch Google News RSS' }, { status: 502 });
}
const rssXml = await rssRes.text();
// Parse RSS items
const items: Array<{ title: string; link: string; pubDate: string | null; outlet: string }> = [];
const itemBlocks = rssXml.match(/<item>([\s\S]*?)<\/item>/gi) || [];
for (const block of itemBlocks.slice(0, 15)) {
const title = cleanTitle(extractTag(block, 'title') || '');
const link = extractTag(block, 'link') || '';
const pubDate = extractTag(block, 'pubDate');
const src = deriveSourceName(block);
if (title && link) items.push({ title, link, pubDate, outlet: src });
}
// 2. Fetch articles, extract author, verify match
const matchedArticles: Array<{
title: string; url: string; outlet: string; pubDate: string | null;
author: string; content: string; analysis: ArticleAnalysis | null;
}> = [];
for (const item of items) {
try {
const realUrl = await resolveGoogleNewsUrl(item.link);
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), 8000);
const res = await fetch(realUrl, {
signal: ctrl.signal,
headers: { 'User-Agent': UA, 'Accept': 'text/html' },
redirect: 'follow',
});
clearTimeout(timer);
if (!res.ok) continue;
const html = await res.text();
const author = extractAuthorFromHtml(html);
if (!author || !nameMatches(author, reporterName)) continue;
const content = extractArticleText(html);
const analysis = await analyzeArticle(item.title, item.outlet, content);
matchedArticles.push({
title: item.title,
url: realUrl,
outlet: item.outlet,
pubDate: item.pubDate,
author,
content,
analysis,
});
} catch {
continue; // Skip failed articles
}
}
// 3. Upsert journalist
const primaryOutlet = outlet || (matchedArticles.length > 0
? matchedArticles.reduce((acc, a) => {
acc[a.outlet] = (acc[a.outlet] || 0) + 1;
return acc;
}, {} as Record<string, number>)
: {});
const outletName = typeof primaryOutlet === 'string'
? primaryOutlet
: Object.entries(primaryOutlet as Record<string, number>).sort((a, b) => b[1] - a[1])[0]?.[0] || '';
// Aggregate all topics for beat detection
const allTopics = matchedArticles.flatMap(a => a.analysis?.topics || []);
const beat = allTopics.length > 0 ? allTopics[0] : 'General';
const tags = [...new Set(matchedArticles.flatMap(a => a.analysis?.tags || []))].slice(0, 10);
const journalistResult = await query(
`INSERT INTO journalists (name, outlet, beat, tags, added_by)
VALUES ($1, $2, $3, $4, 'scraper')
ON CONFLICT (LOWER(name), LOWER(outlet))
DO UPDATE SET updated_at = NOW(), beat = COALESCE(NULLIF($3, 'General'), journalists.beat),
tags = CASE WHEN array_length($4::text[], 1) > 0 THEN $4 ELSE journalists.tags END
RETURNING *`,
[reporterName, outletName, beat, tags],
);
const journalist = journalistResult.rows[0];
const journalistId = journalist.id as string;
// 4. Upsert articles
let articlesInserted = 0;
for (const art of matchedArticles) {
const analysis = art.analysis;
try {
const result = await query(
`INSERT INTO journalist_articles
(journalist_id, title, url, outlet, published_date, summary, tags, topics,
sentiment, word_count, mentioned_people, mentioned_orgs, mentioned_topics, raw_content)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (url) WHERE url IS NOT NULL
DO UPDATE SET summary = EXCLUDED.summary, topics = EXCLUDED.topics,
sentiment = EXCLUDED.sentiment, mentioned_people = EXCLUDED.mentioned_people,
mentioned_orgs = EXCLUDED.mentioned_orgs, mentioned_topics = EXCLUDED.mentioned_topics
RETURNING id`,
[
journalistId,
art.title,
art.url,
art.outlet,
art.pubDate ? new Date(art.pubDate) : null,
analysis?.summary || '',
analysis?.tags || [],
analysis?.topics || [],
analysis?.sentiment || 'neutral',
art.content.split(/\s+/).length,
analysis?.mentioned_people || [],
analysis?.mentioned_orgs || [],
analysis?.topics || [], // mentioned_topics = topics for now
art.content.slice(0, 5000),
],
);
if (result.rowCount && result.rowCount > 0) articlesInserted++;
} catch (err) {
console.error('[journalists/scrape] Article insert error:', (err as Error).message);
}
}
// 5. Aggregate org relationships
const orgCounts: Record<string, number> = {};
for (const art of matchedArticles) {
for (const org of art.analysis?.mentioned_orgs || []) {
const key = org.trim();
if (key.length > 1) orgCounts[key] = (orgCounts[key] || 0) + 1;
}
}
const orgRows: Array<{ org_name: string; strength: number }> = [];
for (const [orgName, count] of Object.entries(orgCounts)) {
const strength = Math.min(5, count);
try {
await query(
`INSERT INTO journalist_orgs (journalist_id, org_name, org_type, relationship, strength)
VALUES ($1, $2, 'organization', 'covers', $3)
ON CONFLICT (journalist_id, LOWER(org_name))
DO UPDATE SET strength = GREATEST(journalist_orgs.strength, $3)`,
[journalistId, orgName, strength],
);
orgRows.push({ org_name: orgName, strength });
} catch (err) {
console.error('[journalists/scrape] Org insert error:', (err as Error).message);
}
}
// 6. Create mind_map_links
const linksCreated = await createJournalistLinks(journalistId);
return NextResponse.json({
journalist,
articles: matchedArticles.map(a => ({
title: a.title,
url: a.url,
outlet: a.outlet,
author: a.author,
summary: a.analysis?.summary,
topics: a.analysis?.topics,
sentiment: a.analysis?.sentiment,
mentioned_people: a.analysis?.mentioned_people,
mentioned_orgs: a.analysis?.mentioned_orgs,
})),
orgs: orgRows,
stats: {
rss_items: items.length,
articles_fetched: items.length,
articles_matched: matchedArticles.length,
articles_analyzed: matchedArticles.filter(a => a.analysis).length,
articles_inserted: articlesInserted,
orgs_found: orgRows.length,
links_created: linksCreated,
},
});
} catch (err) {
console.error('[journalists/scrape] Error:', (err as Error).message);
return NextResponse.json({ error: (err as Error).message }, { status: 500 });
}
}