`agents/datasource-agent/server.js`

128 lines
/**
 * Norma Data Source Agent — Port 9807
 *
 * Discovers data APIs, scrapes newspaper front pages, collects non-profit
 * statements, and rates all sources for advocacy relevance.
 *
 * Skills: discover (discover-apis), post (scrape-frontpages),
 *         reply (collect-statements), monitor (rate-sources), report
 */

const { createAgentServer } = require('../shared/agent-base');

// Import skills
const discover = require('./skills/discover-apis');
const post = require('./skills/scrape-frontpages');
const reply = require('./skills/collect-statements');
const monitor = require('./skills/rate-sources');
const report = require('./skills/report');
const { scrapePetitions } = require('./skills/scrape-petitions');

const PORT = 9807;
const AGENT_NAME = 'datasource-agent';

const { app, start, scheduler } = createAgentServer({
  name: AGENT_NAME,
  port: PORT,
  skills: {
    discover,   // discover-apis
    post,       // scrape-frontpages
    reply,      // collect-statements
    monitor,    // rate-sources
    report,     // report
  },
  cronJobs: [
    {
      name: 'discover-apis',
      schedule: '0 6,14,22 * * *', // 3x/day at 6am, 2pm, 10pm
      fn: async () => {
        console.log(`[${AGENT_NAME}] Cron: discovering APIs...`);
        try {
          const result = await discover({ score: true });
          console.log(`[${AGENT_NAME}] Cron discover-apis complete: ${result.discovered} discovered, ${result.scored} scored`);
        } catch (err) {
          console.error(`[${AGENT_NAME}] Cron discover-apis error:`, err.message);
        }
      },
    },
    {
      name: 'scrape-frontpages',
      schedule: '0 7,12,18 * * *', // 3x/day at 7am, 12pm, 6pm
      fn: async () => {
        console.log(`[${AGENT_NAME}] Cron: scraping front pages...`);
        try {
          const result = await post({ analyze: true });
          console.log(`[${AGENT_NAME}] Cron scrape-frontpages complete: ${result.scraped} scraped, ${result.relevant} relevant`);
        } catch (err) {
          console.error(`[${AGENT_NAME}] Cron scrape-frontpages error:`, err.message);
        }
      },
    },
    {
      name: 'collect-statements',
      schedule: '30 */4 * * *', // every 4 hours at :30
      fn: async () => {
        console.log(`[${AGENT_NAME}] Cron: collecting statements...`);
        try {
          const result = await reply({ analyze: true });
          console.log(`[${AGENT_NAME}] Cron collect-statements complete: ${result.orgs_checked} orgs, ${result.statements_found} statements`);
        } catch (err) {
          console.error(`[${AGENT_NAME}] Cron collect-statements error:`, err.message);
        }
      },
    },
    {
      name: 'rate-sources',
      schedule: '0 */2 * * *', // every 2 hours
      fn: async () => {
        console.log(`[${AGENT_NAME}] Cron: rating sources...`);
        try {
          const result = await monitor({});
          console.log(`[${AGENT_NAME}] Cron rate-sources complete: ${result.total_rated} rated`);
        } catch (err) {
          console.error(`[${AGENT_NAME}] Cron rate-sources error:`, err.message);
        }
      },
    },
    {
      name: 'scrape-petitions',
      schedule: '15 8,16 * * *', // 2x/day at 8:15am and 4:15pm
      fn: async () => {
        console.log(`[${AGENT_NAME}] Cron: scraping petitions...`);
        try {
          const result = await scrapePetitions({ score: true });
          console.log(`[${AGENT_NAME}] Cron scrape-petitions complete: ${result.total_found} found, ${result.total_upserted} upserted`);
        } catch (err) {
          console.error(`[${AGENT_NAME}] Cron scrape-petitions error:`, err.message);
        }
      },
    },
  ],
});

// ──────────────────────────────────────
// Custom skill route: scrape-petitions
// The agent-base registers a catch-all 404 and error handler at the end
// of the router stack. We pop those, add our custom route, then re-add them
// so that our route is matched before the catch-all.
// ──────────────────────────────────────
const stack = app._router.stack;
const errorHandler = stack.pop(); // error handler (last)
const notFoundHandler = stack.pop(); // 404 catch-all (second to last)

app.post('/api/skill/scrape-petitions', async (req, res) => {
  try {
    const result = await scrapePetitions(req.body || {});
    res.json({ success: true, skill: 'scrape-petitions', result });
  } catch (err) {
    console.error(`[${AGENT_NAME}] Skill "scrape-petitions" error:`, err.message);
    res.status(500).json({ success: false, skill: 'scrape-petitions', error: err.message });
  }
});

// Re-add 404 and error handlers at the end
if (notFoundHandler) stack.push(notFoundHandler);
if (errorHandler) stack.push(errorHandler);

start();