`agents/crawler-agent/server.js`

82 lines
#!/usr/bin/env node
/**
 * Crawler agent — HTTP control plane for hospital site crawlers.
 *
 *   GET /health           — { ok, agent, crawlers }
 *   GET /crawlers         — list of crawler module ids that exist on disk
 *   POST /run/:id         — async-fire crawlers/<id>.js
 */
const fs = require('node:fs');
const path = require('path');
const { spawn } = require('node:child_process');
require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
const express = require('express');
const { pool, query } = require('../shared/db');

const PORT = Number(process.env.PORT_CRAWLER || process.env.PORT || 9871);
const CRAWLERS_DIR = path.join(__dirname, 'crawlers');
const app = express();
app.use(express.json());

// LAN/tailnet auth gate (skip /health for watchdog probes)
const BASIC_AUTH = process.env.BASIC_AUTH || 'admin:DWSecure2024!';
const AUTH_HEADER = 'Basic ' + Buffer.from(BASIC_AUTH).toString('base64');
app.use((req, res, next) => {
  if (req.path === '/health' || req.path === '/healthz') return next();
  if (req.get('authorization') === AUTH_HEADER) return next();
  res.set('WWW-Authenticate', 'Basic realm="mac2-pm2"');
  res.status(401).send('auth required');
});

function listCrawlers() {
  if (!fs.existsSync(CRAWLERS_DIR)) return [];
  return fs.readdirSync(CRAWLERS_DIR)
    .filter(f => f.endsWith('.js'))
    .map(f => f.replace(/\.js$/, ''));
}

app.get('/health', async (_req, res) => {
  try {
    await query('SELECT 1', []);
    res.json({ ok: true, agent: 'pd-crawler', crawlers: listCrawlers() });
  } catch (e) {
    res.status(500).json({ ok: false, error: e.message });
  }
});

app.get('/crawlers', (_req, res) => {
  res.json({ count: listCrawlers().length, crawlers: listCrawlers() });
});

// SECURITY: was accepting `req.body.env` and merging into the spawned child's
// environment with no auth. An attacker reaching this loopback port (via any
// proxy that forwards) could overwrite NODE_PATH / DATABASE_URL / STRIPE keys
// in the child. Removed env passthrough and added shared-secret token check.
// CRAWLER_RUN_TOKEN must be set in pm2 env; pass via Authorization: Bearer.
const CRAWLER_RUN_TOKEN = process.env.CRAWLER_RUN_TOKEN || '';
app.post('/run/:id', (req, res) => {
  if (!CRAWLER_RUN_TOKEN) {
    return res.status(503).json({ error: 'CRAWLER_RUN_TOKEN not configured on server' });
  }
  const auth = String(req.get('authorization') || '');
  if (auth !== `Bearer ${CRAWLER_RUN_TOKEN}`) {
    return res.status(401).json({ error: 'unauthorized' });
  }
  const id = String(req.params.id);
  if (!listCrawlers().includes(id)) {
    return res.status(404).json({ error: `unknown crawler '${id}'`, available: listCrawlers() });
  }
  const child = spawn('node', [path.join(CRAWLERS_DIR, `${id}.js`)], {
    cwd: path.resolve(__dirname, '../..'),
    env: process.env,   // NO caller-supplied env merge
    detached: true,
    stdio: 'ignore',
  });
  child.unref();
  res.json({ ok: true, started: id, pid: child.pid });
});

app.listen(PORT, '0.0.0.0', () => {
  console.log(`[pd-crawler] listening on http://127.0.0.1:${PORT}`);
});