`monitoring/health-monitor.js`

298 lines
/**
 * Health Monitoring System
 * Continuously monitors application health and sends alerts
 */

import cron from 'node-cron';
import fs from 'fs/promises';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const CONFIG = {
  checkInterval: '*/5 * * * *', // Every 5 minutes
  healthEndpoint: 'http://localhost:7600/api/health',
  alertThresholds: {
    memoryUsagePercent: 85,
    cpuUsagePercent: 80,
    responseTimeMs: 3000,
    errorRate: 5 // errors per minute
  },
  metricsFile: path.join(__dirname, '../logs/metrics.json'),
  slackWebhook: process.env.SLACK_WEBHOOK_URL,
  emailAlert: process.env.EMAIL_ALERTS_TO
};

class HealthMonitor {
  constructor() {
    this.metrics = {
      uptime: 0,
      lastCheck: null,
      status: 'unknown',
      alerts: [],
      history: []
    };
    this.errorCount = 0;
    this.lastErrorReset = Date.now();
  }

  async start() {
    console.log('🔍 Starting Health Monitor...');

    // Run initial check
    await this.checkHealth();

    // Schedule periodic checks
    cron.schedule(CONFIG.checkInterval, async () => {
      await this.checkHealth();
    });

    // Reset error counter every minute
    cron.schedule('* * * * *', () => {
      this.errorCount = 0;
      this.lastErrorReset = Date.now();
    });

    console.log(`✓ Health Monitor active (checking every 5 minutes)`);
  }

  async checkHealth() {
    const startTime = Date.now();

    try {
      const response = await fetch(CONFIG.healthEndpoint, {
        timeout: 5000
      });

      if (!response.ok) {
        throw new Error(`Health check failed with status ${response.status}`);
      }

      const data = await response.json();
      const responseTime = Date.now() - startTime;

      // Update metrics
      this.metrics.status = 'healthy';
      this.metrics.lastCheck = new Date().toISOString();
      this.metrics.uptime = data.uptime;

      // Check thresholds
      await this.checkThresholds(data, responseTime);

      // Store metrics
      await this.storeMetrics(data, responseTime);

      console.log(`✓ Health check passed (${responseTime}ms) - Memory: ${Math.round(data.memory.heapUsed / 1024 / 1024)}MB`);

    } catch (error) {
      this.errorCount++;
      this.metrics.status = 'unhealthy';
      console.error(`✗ Health check failed:`, error.message);

      await this.sendAlert({
        severity: 'critical',
        title: 'Service Health Check Failed',
        message: error.message,
        timestamp: new Date().toISOString()
      });

      // Check if service needs restart
      if (this.errorCount >= 3) {
        await this.attemptAutoRecover();
      }
    }
  }

  async checkThresholds(data, responseTime) {
    const alerts = [];

    // Check response time
    if (responseTime > CONFIG.alertThresholds.responseTimeMs) {
      alerts.push({
        severity: 'warning',
        metric: 'response_time',
        message: `High response time: ${responseTime}ms`,
        value: responseTime,
        threshold: CONFIG.alertThresholds.responseTimeMs
      });
    }

    // Check memory usage
    const memoryUsagePercent = (data.memory.heapUsed / data.memory.heapTotal) * 100;
    if (memoryUsagePercent > CONFIG.alertThresholds.memoryUsagePercent) {
      alerts.push({
        severity: 'warning',
        metric: 'memory_usage',
        message: `High memory usage: ${memoryUsagePercent.toFixed(2)}%`,
        value: memoryUsagePercent,
        threshold: CONFIG.alertThresholds.memoryUsagePercent
      });
    }

    // Check error rate
    if (this.errorCount >= CONFIG.alertThresholds.errorRate) {
      alerts.push({
        severity: 'critical',
        metric: 'error_rate',
        message: `High error rate: ${this.errorCount} errors/min`,
        value: this.errorCount,
        threshold: CONFIG.alertThresholds.errorRate
      });
    }

    // Send alerts if any
    for (const alert of alerts) {
      await this.sendAlert(alert);
    }
  }

  async storeMetrics(data, responseTime) {
    const metric = {
      timestamp: new Date().toISOString(),
      uptime: data.uptime,
      memory: {
        used: Math.round(data.memory.heapUsed / 1024 / 1024),
        total: Math.round(data.memory.heapTotal / 1024 / 1024),
        percent: ((data.memory.heapUsed / data.memory.heapTotal) * 100).toFixed(2)
      },
      cache: data.cache,
      database: data.database,
      websocket: data.websocket,
      responseTime
    };

    // Keep last 288 entries (24 hours at 5-minute intervals)
    this.metrics.history.push(metric);
    if (this.metrics.history.length > 288) {
      this.metrics.history.shift();
    }

    // Save to file
    try {
      await fs.writeFile(
        CONFIG.metricsFile,
        JSON.stringify(this.metrics, null, 2)
      );
    } catch (error) {
      console.error('Failed to save metrics:', error);
    }
  }

  async sendAlert(alert) {
    console.log(`🚨 ALERT [${alert.severity}]: ${alert.message || alert.title}`);

    this.metrics.alerts.push({
      ...alert,
      timestamp: alert.timestamp || new Date().toISOString()
    });

    // Keep only last 100 alerts
    if (this.metrics.alerts.length > 100) {
      this.metrics.alerts.shift();
    }

    // Send to Slack
    if (CONFIG.slackWebhook) {
      await this.sendSlackAlert(alert);
    }

    // Could add email alerts here
    // if (CONFIG.emailAlert) {
    //   await this.sendEmailAlert(alert);
    // }
  }

  async sendSlackAlert(alert) {
    try {
      const color = alert.severity === 'critical' ? 'danger' : 'warning';
      const emoji = alert.severity === 'critical' ? '🔴' : '⚠️';

      const payload = {
        text: `${emoji} Omega Watches Alert`,
        attachments: [{
          color,
          title: alert.title || alert.metric,
          text: alert.message,
          fields: [
            {
              title: 'Severity',
              value: alert.severity,
              short: true
            },
            {
              title: 'Timestamp',
              value: alert.timestamp,
              short: true
            }
          ],
          footer: 'Omega Watch Price History Monitor'
        }]
      };

      await fetch(CONFIG.slackWebhook, {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(payload)
      });
    } catch (error) {
      console.error('Failed to send Slack alert:', error);
    }
  }

  async attemptAutoRecover() {
    console.log('🔄 Attempting auto-recovery...');

    try {
      // Try to restart PM2 process
      const { exec } = await import('child_process');
      const { promisify } = await import('util');
      const execAsync = promisify(exec);

      await execAsync('pm2 restart omega-watches');

      console.log('✓ Service restarted successfully');

      await this.sendAlert({
        severity: 'warning',
        title: 'Auto-Recovery Executed',
        message: 'Service was automatically restarted due to repeated failures'
      });

      this.errorCount = 0;
    } catch (error) {
      console.error('✗ Auto-recovery failed:', error);

      await this.sendAlert({
        severity: 'critical',
        title: 'Auto-Recovery Failed',
        message: `Manual intervention required: ${error.message}`
      });
    }
  }

  async getMetrics() {
    return this.metrics;
  }

  async getStatus() {
    return {
      status: this.metrics.status,
      lastCheck: this.metrics.lastCheck,
      uptime: this.metrics.uptime,
      recentAlerts: this.metrics.alerts.slice(-10)
    };
  }
}

// Export singleton instance
const monitor = new HealthMonitor();

// Start monitoring if run directly
if (import.meta.url === `file://${process.argv[1]}`) {
  monitor.start();
}

export default monitor;