`backend/services/MonitoringService.js`

544 lines
const fs = require('fs')
const path = require('path')
const axios = require('axios')
const moment = require('moment')
const os = require('os')
const User = require('../models/User')
const Session = require('../models/Session')
const Conversation = require('../models/Conversation')

class MonitoringService {
  constructor(io, logger, alertService) {
    this.io = io
    this.logger = logger
    this.alertService = alertService
    this.currentStats = {
      activeUsers: 0,
      activeSessions: 0,
      totalConversations: 0,
      alertsToday: 0,
      systemHealth: 'healthy',
      lastUpdate: new Date()
    }
    this.systemMetrics = {}
    this.isMonitoring = false
    
    // Track real-time user activity
    this.activeUsers = new Set()
    this.activeSessions = new Map()
    this.realtimeStats = {
      messagesPerMinute: 0,
      avgResponseTime: 0,
      errorRate: 0
    }
    
    this.logger.info('MonitoringService initialized')
  }
  
  startMonitoring() {
    if (this.isMonitoring) return
    
    this.isMonitoring = true
    this.logger.info('Starting real-time monitoring')
    
    // Monitor every 30 seconds
    this.monitoringInterval = setInterval(() => {
      this.collectSystemMetrics()
      this.updateUserStats()
      this.broadcastStats()
    }, 30000)
    
    // Monitor Dear Bubbe API health every minute
    this.healthCheckInterval = setInterval(() => {
      this.checkBubbeHealth()
    }, 60000)
    
    // Generate detailed reports every 5 minutes
    this.reportInterval = setInterval(() => {
      this.generateDetailedReport()
    }, 5 * 60000)
    
    // Initial collection
    this.collectSystemMetrics()
    this.updateUserStats()
  }
  
  stopMonitoring() {
    this.isMonitoring = false
    
    if (this.monitoringInterval) clearInterval(this.monitoringInterval)
    if (this.healthCheckInterval) clearInterval(this.healthCheckInterval)
    if (this.reportInterval) clearInterval(this.reportInterval)
    
    this.logger.info('Monitoring stopped')
  }
  
  async collectSystemMetrics() {
    try {
      // System metrics
      const totalMem = os.totalmem()
      const freeMem = os.freemem()
      const usedMem = totalMem - freeMem
      const memUsagePercent = (usedMem / totalMem) * 100
      
      const loadAvg = os.loadavg()
      const cpuCount = os.cpus().length
      const cpuUsage = (loadAvg[0] / cpuCount) * 100
      
      // Process metrics
      const processMemory = process.memoryUsage()
      const uptime = process.uptime()
      
      this.systemMetrics = {
        timestamp: new Date(),
        system: {
          platform: os.platform(),
          arch: os.arch(),
          hostname: os.hostname(),
          uptime: os.uptime(),
          totalMemory: totalMem,
          freeMemory: freeMem,
          memoryUsage: memUsagePercent,
          cpuUsage: Math.min(cpuUsage, 100),
          loadAverage: loadAvg
        },
        process: {
          pid: process.pid,
          uptime: uptime,
          memory: processMemory,
          memoryUsageMB: Math.round(processMemory.rss / 1024 / 1024),
          version: process.version
        }
      }
      
      // Check for high resource usage
      if (memUsagePercent > 90) {
        this.createSystemAlert('high_memory_usage', {
          usage: memUsagePercent,
          threshold: 90
        })
      }
      
      if (cpuUsage > 80) {
        this.createSystemAlert('high_cpu_usage', {
          usage: cpuUsage,
          threshold: 80
        })
      }
      
    } catch (error) {
      this.logger.error('Failed to collect system metrics:', error)
    }
  }
  
  async updateUserStats() {
    try {
      const now = moment()
      const today = now.startOf('day')
      
      // Active users (last 5 minutes)
      const activeUsers = await Session.countDocuments({
        lastActivity: { $gte: moment().subtract(5, 'minutes').toDate() }
      })
      
      // Active sessions (last 30 minutes)
      const activeSessions = await Session.countDocuments({
        lastActivity: { $gte: moment().subtract(30, 'minutes').toDate() },
        ended: { $ne: true }
      })
      
      // Total conversations today
      const conversationsToday = await Conversation.countDocuments({
        timestamp: { $gte: today.toDate() }
      })
      
      // Alerts today
      const alertsToday = await this.alertService.Alert?.countDocuments?.({
        timestamp: { $gte: today.toDate() }
      }) || 0
      
      // User registrations today
      const registrationsToday = await User.countDocuments({
        createdAt: { $gte: today.toDate() }
      })
      
      // Update current stats
      this.currentStats = {
        activeUsers,
        activeSessions,
        totalConversations: conversationsToday,
        alertsToday,
        registrationsToday,
        systemHealth: this.calculateSystemHealth(),
        lastUpdate: new Date()
      }
      
    } catch (error) {
      this.logger.error('Failed to update user stats:', error)
    }
  }
  
  calculateSystemHealth() {
    const { system } = this.systemMetrics
    if (!system) return 'unknown'
    
    const memUsage = system.memoryUsage || 0
    const cpuUsage = system.cpuUsage || 0
    
    if (memUsage > 90 || cpuUsage > 90) return 'critical'
    if (memUsage > 70 || cpuUsage > 70) return 'warning'
    return 'healthy'
  }
  
  async checkBubbeHealth() {
    try {
      const startTime = Date.now()
      const response = await axios.get('http://localhost:3011/api/health', {
        timeout: 10000
      })
      const responseTime = Date.now() - startTime
      
      if (response.status === 200) {
        this.logger.debug('Dear Bubbe API health check passed')
        this.realtimeStats.avgResponseTime = responseTime
      } else {
        this.createSystemAlert('api_health_failed', {
          status: response.status,
          responseTime
        })
      }
    } catch (error) {
      this.logger.error('Dear Bubbe API health check failed:', error.message)
      this.createSystemAlert('api_unreachable', {
        error: error.message,
        timestamp: new Date()
      })
    }
  }
  
  async generateDetailedReport() {
    try {
      // Get hourly stats for the last 24 hours
      const hourlyStats = await this.getHourlyStats()
      
      // Get top active users
      const topUsers = await this.getTopActiveUsers()
      
      // Get conversation topics breakdown
      const topicBreakdown = await this.getConversationTopics()
      
      // Get geographic distribution
      const geoStats = await this.getGeographicStats()
      
      // Alert trends
      const alertTrends = await this.alertService.getAlertStats('24h')
      
      const report = {
        timestamp: new Date(),
        period: 'last_24_hours',
        systemMetrics: this.systemMetrics,
        userActivity: {
          hourlyStats,
          topUsers,
          topicBreakdown,
          geoStats
        },
        alertTrends,
        realtimeStats: this.realtimeStats
      }
      
      // Broadcast to admin clients
      this.io.to('admin').emit('detailed_report', report)
      
      // Save report to file for historical analysis
      this.saveReportToFile(report)
      
    } catch (error) {
      this.logger.error('Failed to generate detailed report:', error)
    }
  }
  
  async getHourlyStats() {
    try {
      const last24Hours = moment().subtract(24, 'hours')
      
      const hourlyConversations = await Conversation.aggregate([
        {
          $match: {
            timestamp: { $gte: last24Hours.toDate() }
          }
        },
        {
          $group: {
            _id: {
              year: { $year: '$timestamp' },
              month: { $month: '$timestamp' },
              day: { $dayOfMonth: '$timestamp' },
              hour: { $hour: '$timestamp' }
            },
            count: { $sum: 1 },
            uniqueUsers: { $addToSet: '$userId' }
          }
        },
        {
          $addFields: {
            uniqueUserCount: { $size: '$uniqueUsers' }
          }
        },
        {
          $sort: { '_id.year': 1, '_id.month': 1, '_id.day': 1, '_id.hour': 1 }
        }
      ])
      
      return hourlyConversations
    } catch (error) {
      this.logger.error('Failed to get hourly stats:', error)
      return []
    }
  }
  
  async getTopActiveUsers(limit = 20) {
    try {
      const last24Hours = moment().subtract(24, 'hours')
      
      const topUsers = await Conversation.aggregate([
        {
          $match: {
            timestamp: { $gte: last24Hours.toDate() }
          }
        },
        {
          $group: {
            _id: '$userId',
            messageCount: { $sum: 1 },
            lastActivity: { $max: '$timestamp' },
            topics: { $addToSet: '$topics' }
          }
        },
        {
          $sort: { messageCount: -1 }
        },
        {
          $limit: limit
        }
      ])
      
      return topUsers
    } catch (error) {
      this.logger.error('Failed to get top active users:', error)
      return []
    }
  }
  
  async getConversationTopics() {
    try {
      const last24Hours = moment().subtract(24, 'hours')
      
      const topics = await Conversation.aggregate([
        {
          $match: {
            timestamp: { $gte: last24Hours.toDate() }
          }
        },
        {
          $unwind: '$topics'
        },
        {
          $group: {
            _id: '$topics',
            count: { $sum: 1 }
          }
        },
        {
          $sort: { count: -1 }
        },
        {
          $limit: 10
        }
      ])
      
      return topics
    } catch (error) {
      this.logger.error('Failed to get conversation topics:', error)
      return []
    }
  }
  
  async getGeographicStats() {
    try {
      const last24Hours = moment().subtract(24, 'hours')
      
      const geoStats = await Session.aggregate([
        {
          $match: {
            startTime: { $gte: last24Hours.toDate() }
          }
        },
        {
          $group: {
            _id: {
              country: '$location.country',
              state: '$location.state'
            },
            sessions: { $sum: 1 },
            uniqueUsers: { $addToSet: '$userId' }
          }
        },
        {
          $addFields: {
            userCount: { $size: '$uniqueUsers' }
          }
        },
        {
          $sort: { sessions: -1 }
        }
      ])
      
      return geoStats
    } catch (error) {
      this.logger.error('Failed to get geographic stats:', error)
      return []
    }
  }
  
  saveReportToFile(report) {
    try {
      const reportsDir = path.join(__dirname, '../reports')
      if (!fs.existsSync(reportsDir)) {
        fs.mkdirSync(reportsDir, { recursive: true })
      }
      
      const filename = `report_${moment().format('YYYY-MM-DD_HH-mm-ss')}.json`
      const filepath = path.join(reportsDir, filename)
      
      fs.writeFileSync(filepath, JSON.stringify(report, null, 2))
      
      // Clean up old reports (keep only last 30 days)
      this.cleanupOldReports(reportsDir)
      
    } catch (error) {
      this.logger.error('Failed to save report to file:', error)
    }
  }
  
  cleanupOldReports(reportsDir) {
    try {
      const files = fs.readdirSync(reportsDir)
      const cutoff = moment().subtract(30, 'days')
      
      files.forEach(file => {
        const filepath = path.join(reportsDir, file)
        const stats = fs.statSync(filepath)
        
        if (moment(stats.birthtime).isBefore(cutoff)) {
          fs.unlinkSync(filepath)
          this.logger.info(`Deleted old report: ${file}`)
        }
      })
    } catch (error) {
      this.logger.error('Failed to cleanup old reports:', error)
    }
  }
  
  async cleanupOldLogs() {
    try {
      const logsDir = path.join(__dirname, '../logs')
      if (!fs.existsSync(logsDir)) return
      
      const files = fs.readdirSync(logsDir)
      const cutoff = moment().subtract(7, 'days') // Keep logs for 7 days
      
      files.forEach(file => {
        const filepath = path.join(logsDir, file)
        const stats = fs.statSync(filepath)
        
        if (moment(stats.birthtime).isBefore(cutoff)) {
          fs.unlinkSync(filepath)
          this.logger.info(`Deleted old log: ${file}`)
        }
      })
    } catch (error) {
      this.logger.error('Failed to cleanup old logs:', error)
    }
  }
  
  createSystemAlert(type, details) {
    const alert = {
      type: 'system',
      subtype: type,
      severity: this.getAlertSeverity(type),
      timestamp: new Date(),
      details,
      systemMetrics: this.systemMetrics
    }
    
    // Emit to admin dashboard
    this.io.to('admin').emit('system_alert', alert)
    
    this.logger.warn(`System alert: ${type}`, details)
  }
  
  getAlertSeverity(type) {
    const severityMap = {
      'high_memory_usage': 'high',
      'high_cpu_usage': 'high',
      'api_health_failed': 'medium',
      'api_unreachable': 'critical'
    }
    return severityMap[type] || 'medium'
  }
  
  broadcastStats() {
    this.io.to('admin').emit('stats', {
      ...this.currentStats,
      systemMetrics: this.systemMetrics,
      realtimeStats: this.realtimeStats
    })
  }
  
  getCurrentStats() {
    return {
      ...this.currentStats,
      systemMetrics: this.systemMetrics,
      realtimeStats: this.realtimeStats
    }
  }
  
  trackUserActivity(userId, sessionId, activity) {
    this.activeUsers.add(userId)
    
    if (!this.activeSessions.has(sessionId)) {
      this.activeSessions.set(sessionId, {
        userId,
        startTime: new Date(),
        lastActivity: new Date(),
        messageCount: 0
      })
    }
    
    const session = this.activeSessions.get(sessionId)
    session.lastActivity = new Date()
    session.messageCount++
    
    // Clean up inactive sessions (older than 30 minutes)
    const cutoff = moment().subtract(30, 'minutes')
    for (const [sid, sessionData] of this.activeSessions.entries()) {
      if (moment(sessionData.lastActivity).isBefore(cutoff)) {
        this.activeSessions.delete(sid)
        this.activeUsers.delete(sessionData.userId)
      }
    }
    
    // Update real-time stats
    this.currentStats.activeUsers = this.activeUsers.size
    this.currentStats.activeSessions = this.activeSessions.size
  }
  
  async generateRealtimeStats() {
    // This method is called every 5 minutes to update real-time statistics
    await this.updateUserStats()
    this.broadcastStats()
  }
}

module.exports = MonitoringService