`agents/datasource-agent/skills/scrape-frontpages.js`

258 lines
/**
 * Scrape Frontpages Skill
 *
 * Scrapes newspaper RSS feeds for today's headlines, parses them,
 * and uses Gemini 2.0 Flash to analyze topics, sentiment, and advocacy relevance.
 * Results are stored in the newspaper_frontpages table.
 */

const fetch = require('node-fetch');
const { logAction } = require('../../shared/audit-logger');
const { ALL_NEWSPAPERS } = require('../lib/newspaper-registry');
const { parseRSS } = require('../lib/frontpage-parser');

const AGENT = 'datasource-agent';
const PLATFORM = 'newspapers';

const GEMINI_URL =
  'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GOOGLE_API_KEY}';

const DB_URL = process.env.DATABASE_URL || 'postgresql://dw_admin@127.0.0.1:5432/sdcc';

let _pool;
function getPool() {
  if (!_pool) {
    const { Pool } = require('pg');
    _pool = new Pool({ connectionString: DB_URL, max: 5 });
  }
  return _pool;
}

/**
 * Fetch and parse an RSS feed with timeout.
 *
 * @param {string} url - RSS feed URL
 * @returns {Promise<Array>} Parsed items
 */
async function fetchFeed(url) {
  const controller = new AbortController();
  const timeout = setTimeout(() => controller.abort(), 15000);
  try {
    const res = await fetch(url, {
      signal: controller.signal,
      headers: {
        'User-Agent': 'Norma-DataSource-Agent/1.0 (research; non-commercial)',
        'Accept': 'application/rss+xml, application/xml, text/xml, */*',
      },
    });
    clearTimeout(timeout);
    if (!res.ok) return [];
    const xml = await res.text();
    return parseRSS(xml);
  } catch (err) {
    clearTimeout(timeout);
    return [];
  }
}

/**
 * Analyze headlines with Gemini for advocacy relevance.
 *
 * @param {string} newspaperName - Name of the newspaper
 * @param {Array} headlines - Array of headline objects
 * @returns {Promise<{ topics: string[], sentiment: string, relevance_score: number, top_headline: string }>}
 */
async function geminiAnalyzeHeadlines(newspaperName, headlines) {
  if (!headlines || headlines.length === 0) {
    return { topics: [], sentiment: 'neutral', relevance_score: 0, top_headline: '' };
  }

  const headlineTexts = headlines.slice(0, 20).map((h, i) => `${i + 1}. ${h.title}`).join('\n');

  try {
    const prompt = `Analyze these headlines from ${newspaperName} for relevance to nonprofit advocacy, education policy, and economic inequality.

Headlines:
${headlineTexts}

Respond in EXACTLY this JSON format (no markdown, no code fences):
{"topics": ["topic1", "topic2"], "sentiment": "positive|negative|neutral|mixed", "relevance_score": 0-100, "top_headline": "most relevant headline text or first headline if none relevant", "reasoning": "brief explanation"}

Score 0 if no headlines relate to education/advocacy/economy. Score higher for direct nonprofit or civic advocacy mentions.`;

    const res = await fetch(GEMINI_URL, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        contents: [{ parts: [{ text: prompt }] }],
        generationConfig: { temperature: 0.2, maxOutputTokens: 500 },
      }),
    });

    const data = await res.json();
    const text = data?.candidates?.[0]?.content?.parts?.[0]?.text || '';

    const jsonMatch = text.match(/\{[\s\S]*\}/);
    if (jsonMatch) {
      const parsed = JSON.parse(jsonMatch[0]);
      return {
        topics: Array.isArray(parsed.topics) ? parsed.topics : [],
        sentiment: parsed.sentiment || 'neutral',
        relevance_score: Math.min(100, Math.max(0, Number(parsed.relevance_score) || 0)),
        top_headline: (parsed.top_headline || headlines[0]?.title || '').substring(0, 500),
      };
    }

    return {
      topics: [],
      sentiment: 'neutral',
      relevance_score: 0,
      top_headline: headlines[0]?.title || '',
    };
  } catch (err) {
    console.error(`[${AGENT}] Gemini analysis error for ${newspaperName}:`, err.message);
    return {
      topics: [],
      sentiment: 'neutral',
      relevance_score: 0,
      top_headline: headlines[0]?.title || '',
    };
  }
}

/**
 * Insert/update frontpage data in the database.
 *
 * @param {Object} newspaper - Newspaper registry entry
 * @param {Array} headlines - Parsed headline items
 * @param {Object} analysis - Gemini analysis result
 * @returns {Promise<Object|null>}
 */
async function upsertFrontpage(newspaper, headlines, analysis) {
  const pool = getPool();
  try {
    const { rows } = await pool.query(
      `INSERT INTO newspaper_frontpages
        (newspaper_name, newspaper_url, country, region, frontpage_url, headlines, top_headline, scraped_date, analysis, relevance_score, match_topics, source_type)
       VALUES ($1, $2, $3, $4, $5, $6, $7, CURRENT_DATE, $8, $9, $10, $11)
       ON CONFLICT (newspaper_name, scraped_date) DO UPDATE SET
         headlines = EXCLUDED.headlines,
         top_headline = EXCLUDED.top_headline,
         analysis = EXCLUDED.analysis,
         relevance_score = EXCLUDED.relevance_score,
         match_topics = EXCLUDED.match_topics
       RETURNING id, newspaper_name, relevance_score`,
      [
        newspaper.name,
        newspaper.url,
        newspaper.country,
        newspaper.region || null,
        newspaper.url,
        JSON.stringify(headlines.slice(0, 30)),
        analysis.top_headline,
        JSON.stringify({ topics: analysis.topics, sentiment: analysis.sentiment }),
        analysis.relevance_score,
        analysis.topics,
        newspaper.source_type,
      ]
    );
    return rows[0] || null;
  } catch (err) {
    console.error(`[${AGENT}] DB upsert error for "${newspaper.name}":`, err.message);
    return null;
  }
}

/**
 * Main scrape-frontpages skill handler.
 *
 * @param {Object} body - Request body
 * @param {boolean} [body.analyze=true] - Whether to AI-analyze headlines
 * @param {number} [body.max_papers] - Limit number of papers to scrape
 * @param {string} [body.source_type] - Filter: 'domestic' or 'international'
 * @returns {Promise<Object>} Scrape results
 */
module.exports = async function scrapeFrontpages(body = {}) {
  const shouldAnalyze = body.analyze !== false;
  const sourceTypeFilter = body.source_type || null;

  let papers = ALL_NEWSPAPERS;
  if (sourceTypeFilter) {
    papers = papers.filter((p) => p.source_type === sourceTypeFilter);
  }
  if (body.max_papers) {
    papers = papers.slice(0, body.max_papers);
  }

  let scraped = 0;
  let relevant = 0;
  let failed = 0;
  const relevantPapers = [];

  for (const paper of papers) {
    console.log(`[${AGENT}] Scraping RSS: ${paper.name}...`);

    const headlines = await fetchFeed(paper.url);

    if (headlines.length === 0) {
      failed++;
      continue;
    }

    let analysis = {
      topics: [],
      sentiment: 'neutral',
      relevance_score: 0,
      top_headline: headlines[0]?.title || '',
    };

    if (shouldAnalyze) {
      analysis = await geminiAnalyzeHeadlines(paper.name, headlines);
      // Small delay to avoid Gemini rate limits
      await new Promise((r) => setTimeout(r, 250));
    }

    const result = await upsertFrontpage(paper, headlines, analysis);
    if (result) {
      scraped++;
      if (analysis.relevance_score >= 30) {
        relevant++;
        relevantPapers.push({
          name: paper.name,
          country: paper.country,
          score: analysis.relevance_score,
          top_headline: analysis.top_headline,
          topics: analysis.topics,
        });
      }
    }
  }

  // Sort by relevance
  relevantPapers.sort((a, b) => b.score - a.score);

  await logAction({
    agent: AGENT,
    actionType: 'scrape',
    platform: PLATFORM,
    content: `Scraped ${scraped} newspaper RSS feeds`,
    responseData: {
      total_papers: papers.length,
      scraped,
      relevant,
      failed,
      top_relevant: relevantPapers.slice(0, 5),
    },
    status: 'success',
  });

  return {
    scraped,
    relevant,
    failed,
    total_papers: papers.length,
    relevant_papers: relevantPapers.slice(0, 10),
    fetched_at: new Date().toISOString(),
  };
};