`backend/routes/monitoring.js`

516 lines
const express = require('express')
const Session = require('../models/Session')
const Conversation = require('../models/Conversation')
const Alert = require('../models/Alert')
const User = require('../models/User')
const os = require('os')
const fs = require('fs')
const path = require('path')
const axios = require('axios')
const router = express.Router()

// Get real-time dashboard stats
router.get('/dashboard', async (req, res) => {
  try {
    const now = new Date()
    const last24h = new Date(now - 24 * 60 * 60 * 1000)
    const lastHour = new Date(now - 60 * 60 * 1000)
    const last5min = new Date(now - 5 * 60 * 1000)
    
    // Active users and sessions
    const activeUsers = await Session.countDocuments({
      lastActivity: { $gte: last5min },
      ended: false
    })
    
    const activeSessions = await Session.countDocuments({
      lastActivity: { $gte: new Date(now - 30 * 60 * 1000) },
      ended: false
    })
    
    // Conversations stats
    const conversationsToday = await Conversation.countDocuments({
      timestamp: { $gte: last24h }
    })
    
    const conversationsLastHour = await Conversation.countDocuments({
      timestamp: { $gte: lastHour }
    })
    
    // Alert stats
    const alertsToday = await Alert.countDocuments({
      timestamp: { $gte: last24h }
    })
    
    const criticalAlerts = await Alert.countDocuments({
      timestamp: { $gte: last24h },
      severity: { $in: ['high', 'critical'] }
    })
    
    // System metrics
    const totalMem = os.totalmem()
    const freeMem = os.freemem()
    const memUsage = ((totalMem - freeMem) / totalMem) * 100
    
    const loadAvg = os.loadavg()
    const cpuUsage = (loadAvg[0] / os.cpus().length) * 100
    
    // Process metrics
    const processMemory = process.memoryUsage()
    
    // User stats
    const totalUsers = await User.countDocuments()
    const newUsersToday = await User.countDocuments({
      createdAt: { $gte: last24h }
    })
    
    const blockedUsers = await User.countDocuments({
      'security.blocked': true
    })
    
    res.json({
      success: true,
      timestamp: now,
      userActivity: {
        activeUsers,
        activeSessions,
        totalUsers,
        newUsersToday,
        blockedUsers
      },
      conversations: {
        today: conversationsToday,
        lastHour: conversationsLastHour
      },
      alerts: {
        today: alertsToday,
        critical: criticalAlerts
      },
      system: {
        memoryUsage: Math.round(memUsage),
        cpuUsage: Math.min(Math.round(cpuUsage), 100),
        uptime: os.uptime(),
        loadAverage: loadAvg[0],
        processMemoryMB: Math.round(processMemory.rss / 1024 / 1024)
      }
    })
    
  } catch (error) {
    console.error('Error fetching dashboard stats:', error)
    res.status(500).json({ 
      success: false, 
      error: 'Failed to fetch dashboard statistics' 
    })
  }
})

// Get hourly activity for the last 24 hours
router.get('/hourly-activity', async (req, res) => {
  try {
    const now = new Date()
    const last24h = new Date(now - 24 * 60 * 60 * 1000)
    
    // Get hourly conversation stats
    const hourlyConversations = await Conversation.aggregate([
      {
        $match: {
          timestamp: { $gte: last24h }
        }
      },
      {
        $group: {
          _id: {
            year: { $year: '$timestamp' },
            month: { $month: '$timestamp' },
            day: { $dayOfMonth: '$timestamp' },
            hour: { $hour: '$timestamp' }
          },
          conversations: { $sum: 1 },
          uniqueUsers: { $addToSet: '$userId' }
        }
      },
      {
        $addFields: {
          userCount: { $size: '$uniqueUsers' },
          hour: '$_id.hour'
        }
      },
      {
        $sort: { '_id.year': 1, '_id.month': 1, '_id.day': 1, '_id.hour': 1 }
      }
    ])
    
    // Get hourly alert stats
    const hourlyAlerts = await Alert.aggregate([
      {
        $match: {
          timestamp: { $gte: last24h }
        }
      },
      {
        $group: {
          _id: {
            year: { $year: '$timestamp' },
            month: { $month: '$timestamp' },
            day: { $dayOfMonth: '$timestamp' },
            hour: { $hour: '$timestamp' }
          },
          alerts: { $sum: 1 },
          critical: { $sum: { $cond: [{ $in: ['$severity', ['high', 'critical']] }, 1, 0] } }
        }
      },
      {
        $sort: { '_id.year': 1, '_id.month': 1, '_id.day': 1, '_id.hour': 1 }
      }
    ])
    
    res.json({
      success: true,
      timeframe: 'last_24_hours',
      conversations: hourlyConversations,
      alerts: hourlyAlerts
    })
    
  } catch (error) {
    console.error('Error fetching hourly activity:', error)
    res.status(500).json({ 
      success: false, 
      error: 'Failed to fetch hourly activity' 
    })
  }
})

// Get geographic distribution
router.get('/geographic', async (req, res) => {
  try {
    const { timeframe = '24h' } = req.query
    
    const now = new Date()
    let startTime
    
    switch (timeframe) {
      case '1h':
        startTime = new Date(now - 60 * 60 * 1000)
        break
      case '24h':
        startTime = new Date(now - 24 * 60 * 60 * 1000)
        break
      case '7d':
        startTime = new Date(now - 7 * 24 * 60 * 60 * 1000)
        break
      default:
        startTime = new Date(now - 24 * 60 * 60 * 1000)
    }
    
    // Get session geographic data
    const geoData = await Session.aggregate([
      {
        $match: {
          startTime: { $gte: startTime }
        }
      },
      {
        $group: {
          _id: {
            country: '$location.country',
            state: '$location.state'
          },
          sessions: { $sum: 1 },
          uniqueUsers: { $addToSet: '$userId' },
          messages: { $sum: '$activity.messageCount' },
          alerts: { $sum: '$flags.alertsTriggered' }
        }
      },
      {
        $addFields: {
          userCount: { $size: '$uniqueUsers' }
        }
      },
      {
        $sort: { sessions: -1 }
      },
      {
        $limit: 50
      }
    ])
    
    // Get country-level stats
    const countryStats = await Session.aggregate([
      {
        $match: {
          startTime: { $gte: startTime }
        }
      },
      {
        $group: {
          _id: '$location.country',
          sessions: { $sum: 1 },
          uniqueUsers: { $addToSet: '$userId' },
          messages: { $sum: '$activity.messageCount' }
        }
      },
      {
        $addFields: {
          userCount: { $size: '$uniqueUsers' }
        }
      },
      {
        $sort: { sessions: -1 }
      },
      {
        $limit: 20
      }
    ])
    
    res.json({
      success: true,
      timeframe,
      detailed: geoData,
      byCountry: countryStats
    })
    
  } catch (error) {
    console.error('Error fetching geographic data:', error)
    res.status(500).json({ 
      success: false, 
      error: 'Failed to fetch geographic data' 
    })
  }
})

// Get topic analysis
router.get('/topics', async (req, res) => {
  try {
    const { timeframe = '24h', limit = 20 } = req.query
    
    const topTopics = await Conversation.getTopTopics(parseInt(limit), timeframe)
    
    // Get sentiment analysis for top topics
    const topicSentiment = await Conversation.aggregate([
      {
        $match: {
          timestamp: { $gte: new Date(Date.now() - 24 * 60 * 60 * 1000) }
        }
      },
      {
        $unwind: '$topics'
      },
      {
        $group: {
          _id: {
            topic: '$topics',
            sentiment: '$message.sentiment'
          },
          count: { $sum: 1 }
        }
      },
      {
        $group: {
          _id: '$_id.topic',
          sentiments: {
            $push: {
              sentiment: '$_id.sentiment',
              count: '$count'
            }
          },
          total: { $sum: '$count' }
        }
      },
      {
        $sort: { total: -1 }
      },
      {
        $limit: 10
      }
    ])
    
    res.json({
      success: true,
      timeframe,
      topTopics,
      sentimentAnalysis: topicSentiment
    })
    
  } catch (error) {
    console.error('Error fetching topic analysis:', error)
    res.status(500).json({ 
      success: false, 
      error: 'Failed to fetch topic analysis' 
    })
  }
})

// Check Dear Bubbe API health
router.get('/bubbe-health', async (req, res) => {
  try {
    const startTime = Date.now()
    
    // Check main API
    const apiResponse = await axios.get('http://localhost:3011/api/health', {
      timeout: 10000
    })
    
    const responseTime = Date.now() - startTime
    
    // Check if PM2 process is running
    let pm2Status = 'unknown'
    try {
      const { execSync } = require('child_process')
      const pm2List = execSync('pm2 jlist', { encoding: 'utf-8' })
      const processes = JSON.parse(pm2List)
      const bubbeProcess = processes.find(p => p.name === 'bubbe')
      
      if (bubbeProcess) {
        pm2Status = bubbeProcess.pm2_env.status
      }
    } catch (error) {
      console.warn('Could not check PM2 status:', error.message)
    }
    
    // Check disk space
    const stats = fs.statSync('/')
    const diskUsage = {
      // This is a simplified check - in production you'd use proper disk space checking
      available: true
    }
    
    res.json({
      success: true,
      timestamp: new Date(),
      api: {
        status: apiResponse.status === 200 ? 'healthy' : 'unhealthy',
        responseTime,
        data: apiResponse.data
      },
      pm2: {
        status: pm2Status
      },
      disk: diskUsage,
      overall: apiResponse.status === 200 && pm2Status === 'online' ? 'healthy' : 'degraded'
    })
    
  } catch (error) {
    console.error('Bubbe health check failed:', error.message)
    res.json({
      success: false,
      timestamp: new Date(),
      api: {
        status: 'unhealthy',
        error: error.message
      },
      overall: 'unhealthy'
    })
  }
})

// Get live session data
router.get('/live-sessions', async (req, res) => {
  try {
    const activeSessions = await Session.getActiveSessions(30) // Last 30 minutes
    
    // Enrich with user data
    const enrichedSessions = await Promise.all(
      activeSessions.map(async (session) => {
        const user = await User.findOne({ userId: session.userId })
          .select('profile.name profile.preferredName security.riskLevel security.blocked')
          .lean()
        
        return {
          ...session,
          user: user || { profile: { name: 'Unknown User' } }
        }
      })
    )
    
    res.json({
      success: true,
      timestamp: new Date(),
      activeSessions: enrichedSessions,
      count: enrichedSessions.length
    })
    
  } catch (error) {
    console.error('Error fetching live sessions:', error)
    res.status(500).json({ 
      success: false, 
      error: 'Failed to fetch live sessions' 
    })
  }
})

// Get performance metrics
router.get('/performance', async (req, res) => {
  try {
    const { timeframe = '24h' } = req.query
    
    const now = new Date()
    let startTime
    
    switch (timeframe) {
      case '1h':
        startTime = new Date(now - 60 * 60 * 1000)
        break
      case '24h':
        startTime = new Date(now - 24 * 60 * 60 * 1000)
        break
      case '7d':
        startTime = new Date(now - 7 * 24 * 60 * 60 * 1000)
        break
      default:
        startTime = new Date(now - 24 * 60 * 60 * 1000)
    }
    
    // Response time stats
    const responseTimeStats = await Conversation.aggregate([
      {
        $match: {
          timestamp: { $gte: startTime },
          'analysis.responseTime': { $exists: true }
        }
      },
      {
        $group: {
          _id: null,
          avgResponseTime: { $avg: '$analysis.responseTime' },
          minResponseTime: { $min: '$analysis.responseTime' },
          maxResponseTime: { $max: '$analysis.responseTime' },
          count: { $sum: 1 }
        }
      }
    ])
    
    // Error rate
    const errorStats = await Session.aggregate([
      {
        $match: {
          startTime: { $gte: startTime }
        }
      },
      {
        $group: {
          _id: null,
          totalSessions: { $sum: 1 },
          totalErrors: { $sum: '$activity.errorsEncountered' },
          avgErrors: { $avg: '$activity.errorsEncountered' }
        }
      }
    ])
    
    res.json({
      success: true,
      timeframe,
      responseTime: responseTimeStats[0] || {},
      errors: errorStats[0] || {},
      errorRate: errorStats[0] ? (errorStats[0].totalErrors / errorStats[0].totalSessions) * 100 : 0
    })
    
  } catch (error) {
    console.error('Error fetching performance metrics:', error)
    res.status(500).json({ 
      success: false, 
      error: 'Failed to fetch performance metrics' 
    })
  }
})

module.exports = router