← back to Designer Wallcoverings

DW-Agents/dw-agents/agent-monitor-dashboard/monitor-dashboard.ts

388 lines

import express from 'express';
import { exec } from 'child_process';
import { promisify } from 'util';
import fs from 'fs/promises';
import path from 'path';
import cookieParser from 'cookie-parser';
import session from 'express-session';
import { requireGlobalAuth } from '../shared-global-auth';

const execAsync = promisify(exec);
const app = express();
const PORT = 9899;

// Middleware
app.use(express.json());
app.use(cookieParser());
app.use(session({
  secret: process.env.SESSION_SECRET || 'dw-monitor-dashboard-secret-2025',
  resave: false,
  saveUninitialized: false,
  cookie: {
    secure: false,
    httpOnly: true,
    maxAge: 7 * 24 * 60 * 60 * 1000 // 7 days
  }
}));
app.use(express.static('public'));

// Apply authentication middleware
app.use(requireGlobalAuth);

// Agent categories for grouping
const AGENT_CATEGORIES = {
  'Executive': ['dw-executive-ceo', 'dw-executive-cfo', 'dw-executive-coo', 'dw-executive-vp-ops'],
  'Operations': ['dw-master-hub', 'dw-control-panel', 'dw-dashboard-monitor', 'dw-crash-loop-resolver'],
  'Customer Service': ['dw-zendesk-chat', 'dw-shopify-store', 'dw-digital-samples', 'dw-needs-attention'],
  'Business Intelligence': ['dw-bigquery-gov', 'dw-trend-research', 'dw-marketing', 'dw-accounting'],
  'Infrastructure': ['dw-server-uptime', 'dw-log-monitor', 'dw-ui-manager', 'dw-legal-team'],
  'Support': ['dw-purchasing-office']
};

// Health thresholds
const THRESHOLDS = {
  RESTART_WARNING: 10,
  RESTART_CRITICAL: 20,
  RESTART_SEVERE: 50,
  MEMORY_WARNING: 200, // MB
  MEMORY_CRITICAL: 400, // MB
  CPU_WARNING: 50, // %
  CPU_CRITICAL: 80, // %
  UPTIME_GOOD: 3600, // 1 hour
  UPTIME_WARNING: 300 // 5 minutes
};

// Types
interface PM2Process {
  name: string;
  pm_id: number;
  pm2_env: {
    status: string;
    restart_time?: number;
    pm_uptime?: number;
    unstable_restarts?: number;
    created_at: number;
    instances?: number;
    exec_mode: string;
  };
  monit?: {
    cpu?: number;
    memory?: number;
  };
}

interface AgentStats {
  name: string;
  status: string;
  restarts: number;
  cpu: number;
  memory: number;
  uptime: number;
  uptimeFormatted: string;
  pm_id: number;
  error_count: number;
  created_at: number;
  instances: number;
  exec_mode: string;
  errorLogs?: string[];
}

// Get PM2 stats
async function getPM2Stats(): Promise<AgentStats[]> {
  try {
    const { stdout } = await execAsync('pm2 jlist');
    const processes: PM2Process[] = JSON.parse(stdout);
    
    return processes.map((proc: PM2Process) => {
      const uptime = proc.pm2_env.pm_uptime ? Date.now() - proc.pm2_env.pm_uptime : 0;
      const uptimeSeconds = uptime / 1000;
      
      return {
        name: proc.name,
        status: proc.pm2_env.status,
        restarts: proc.pm2_env.restart_time || 0,
        cpu: proc.monit?.cpu || 0,
        memory: proc.monit?.memory ? Math.round(proc.monit.memory / 1024 / 1024) : 0, // MB
        uptime: uptimeSeconds,
        uptimeFormatted: formatUptime(uptimeSeconds),
        pm_id: proc.pm_id,
        error_count: proc.pm2_env.unstable_restarts || 0,
        created_at: proc.pm2_env.created_at,
        instances: proc.pm2_env.instances || 1,
        exec_mode: proc.pm2_env.exec_mode
      };
    }).filter((proc: AgentStats) => proc.name.startsWith('dw-')); // Only DW agents
  } catch (error) {
    console.error('Error getting PM2 stats:', error);
    return [];
  }
}

// Get latest error logs
async function getErrorLogs(agentName: string, lines: number = 5) {
  try {
    const logPath = `/root/DW-Agents/logs/${agentName.replace('dw-', '')}-error.log`;
    const fileExists = await fs.access(logPath).then(() => true).catch(() => false);
    
    if (!fileExists) {
      return [];
    }
    
    const { stdout } = await execAsync(`tail -n ${lines} "${logPath}" 2>/dev/null || echo ""`);
    return stdout.split('\n').filter(line => line.trim()).slice(-lines);
  } catch (error) {
    return [];
  }
}

// Format uptime
function formatUptime(seconds: number): string {
  if (seconds < 60) return `${Math.floor(seconds)}s`;
  if (seconds < 3600) return `${Math.floor(seconds / 60)}m ${Math.floor(seconds % 60)}s`;
  if (seconds < 86400) return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
  return `${Math.floor(seconds / 86400)}d ${Math.floor((seconds % 86400) / 3600)}h`;
}

// Calculate health score
function calculateHealthScore(stats: AgentStats[]): number {
  let totalScore = 0;
  let agentCount = 0;
  
  for (const agent of stats) {
    let agentScore = 100;
    
    // Status check
    if (agent.status !== 'online') {
      agentScore -= 50;
    }
    
    // Restart penalty
    if (agent.restarts > THRESHOLDS.RESTART_SEVERE) {
      agentScore -= 30;
    } else if (agent.restarts > THRESHOLDS.RESTART_CRITICAL) {
      agentScore -= 20;
    } else if (agent.restarts > THRESHOLDS.RESTART_WARNING) {
      agentScore -= 10;
    }
    
    // Memory penalty
    if (agent.memory > THRESHOLDS.MEMORY_CRITICAL) {
      agentScore -= 15;
    } else if (agent.memory > THRESHOLDS.MEMORY_WARNING) {
      agentScore -= 5;
    }
    
    // CPU penalty
    if (agent.cpu > THRESHOLDS.CPU_CRITICAL) {
      agentScore -= 15;
    } else if (agent.cpu > THRESHOLDS.CPU_WARNING) {
      agentScore -= 5;
    }
    
    // Uptime bonus
    if (agent.uptime > THRESHOLDS.UPTIME_GOOD) {
      agentScore = Math.min(100, agentScore + 10);
    } else if (agent.uptime < THRESHOLDS.UPTIME_WARNING) {
      agentScore -= 10;
    }
    
    totalScore += Math.max(0, agentScore);
    agentCount++;
  }
  
  return agentCount > 0 ? Math.round(totalScore / agentCount) : 0;
}

// API Endpoints

// Get all agent stats
app.get('/api/stats', async (req, res) => {
  try {
    const stats = await getPM2Stats();
    const healthScore = calculateHealthScore(stats);
    
    // Get error logs for problematic agents
    for (const agent of stats) {
      if (agent.restarts > THRESHOLDS.RESTART_WARNING || agent.status !== 'online') {
        agent.errorLogs = await getErrorLogs(agent.name, 3);
      }
    }
    
    // Categorize agents
    const categorizedStats: Record<string, AgentStats[]> = {};
    for (const [category, agentNames] of Object.entries(AGENT_CATEGORIES)) {
      categorizedStats[category] = stats.filter((s: AgentStats) => agentNames.includes(s.name));
    }
    
    // Find uncategorized agents
    const allCategorizedNames = Object.values(AGENT_CATEGORIES).flat();
    const uncategorized = stats.filter((s: AgentStats) => !allCategorizedNames.includes(s.name));
    if (uncategorized.length > 0) {
      categorizedStats['Other'] = uncategorized;
    }
    
    res.json({
      success: true,
      data: {
        stats,
        categorized: categorizedStats,
        healthScore,
        timestamp: new Date().toISOString(),
        thresholds: THRESHOLDS,
        summary: {
          total: stats.length,
          online: stats.filter((s: AgentStats) => s.status === 'online').length,
          errored: stats.filter((s: AgentStats) => s.status === 'errored').length,
          stopped: stats.filter((s: AgentStats) => s.status === 'stopped').length,
          launching: stats.filter((s: AgentStats) => s.status === 'launching').length,
          criticalRestarts: stats.filter((s: AgentStats) => s.restarts > THRESHOLDS.RESTART_CRITICAL).length,
          highMemory: stats.filter((s: AgentStats) => s.memory > THRESHOLDS.MEMORY_WARNING).length,
          highCpu: stats.filter((s: AgentStats) => s.cpu > THRESHOLDS.CPU_WARNING).length
        }
      }
    });
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// Restart agent
app.post('/api/restart/:name', async (req, res) => {
  try {
    const { name } = req.params;
    const { stdout } = await execAsync(`pm2 restart ${name}`);
    res.json({ success: true, message: `Agent ${name} restarted`, output: stdout });
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// Stop agent
app.post('/api/stop/:name', async (req, res) => {
  try {
    const { name } = req.params;
    const { stdout } = await execAsync(`pm2 stop ${name}`);
    res.json({ success: true, message: `Agent ${name} stopped`, output: stdout });
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// Delete agent
app.post('/api/delete/:name', async (req, res) => {
  try {
    const { name } = req.params;
    const { stdout } = await execAsync(`pm2 delete ${name}`);
    res.json({ success: true, message: `Agent ${name} deleted`, output: stdout });
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// View full logs
app.get('/api/logs/:name', async (req, res) => {
  try {
    const { name } = req.params;
    const { lines = 100 } = req.query;
    
    const errorLogs = await getErrorLogs(name, Number(lines));
    const { stdout: outputLogs } = await execAsync(`pm2 logs ${name} --nostream --lines ${lines} 2>/dev/null || echo ""`);
    
    res.json({
      success: true,
      data: {
        errorLogs,
        outputLogs: outputLogs.split('\n').filter(line => line.trim()),
        agent: name
      }
    });
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// Export health report
app.get('/api/export', async (req, res) => {
  try {
    const stats = await getPM2Stats();
    const healthScore = calculateHealthScore(stats);
    const timestamp = new Date().toISOString();
    
    const report = {
      timestamp,
      healthScore,
      summary: {
        total: stats.length,
        online: stats.filter((s: AgentStats) => s.status === 'online').length,
        errored: stats.filter((s: AgentStats) => s.status === 'errored').length,
        stopped: stats.filter((s: AgentStats) => s.status === 'stopped').length,
        criticalRestarts: stats.filter((s: AgentStats) => s.restarts > THRESHOLDS.RESTART_CRITICAL).length
      },
      agents: stats.map((s: AgentStats) => ({
        name: s.name,
        status: s.status,
        restarts: s.restarts,
        cpu: s.cpu,
        memory: s.memory,
        uptime: s.uptimeFormatted,
        issues: ([] as string[])
          .concat(s.restarts > THRESHOLDS.RESTART_CRITICAL ? [`High restarts: ${s.restarts}`] : [])
          .concat(s.memory > THRESHOLDS.MEMORY_WARNING ? [`High memory: ${s.memory}MB`] : [])
          .concat(s.cpu > THRESHOLDS.CPU_WARNING ? [`High CPU: ${s.cpu}%`] : [])
          .concat(s.status !== 'online' ? [`Status: ${s.status}`] : [])
      }))
    };
    
    res.setHeader('Content-Type', 'application/json');
    res.setHeader('Content-Disposition', `attachment; filename="dw-agents-health-report-${Date.now()}.json"`);
    res.send(JSON.stringify(report, null, 2));
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// Batch restart problematic agents
app.post('/api/restart-problematic', async (req, res) => {
  try {
    const stats = await getPM2Stats();
    const problematic = stats.filter((s: AgentStats) => 
      s.restarts > THRESHOLDS.RESTART_CRITICAL || 
      s.status === 'errored' ||
      s.status === 'launching'
    );
    
    const results = [];
    for (const agent of problematic) {
      try {
        await execAsync(`pm2 restart ${agent.name}`);
        results.push({ agent: agent.name, success: true });
      } catch (error) {
        results.push({ agent: agent.name, success: false, error: String(error) });
      }
    }
    
    res.json({ success: true, restarted: results });
  } catch (error: any) {
    res.status(500).json({ success: false, error: error.message });
  }
});

// Serve dashboard HTML
app.get('/', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'index.html'));
});

// Start server
app.listen(PORT, () => {
  console.log(`
╔═══════════════════════════════════════════════════════════╗
║         DW-AGENTS MONITOR DASHBOARD                       ║
║         Real-time Health Monitoring System                ║
╠═══════════════════════════════════════════════════════════╣
║  🚀 Server running on port ${PORT}                          ║
║  📊 Dashboard URL: http://45.61.58.125:${PORT}              ║
║  🔄 Auto-refresh: Every 10 seconds                        ║
║  ⚠️  Monitoring ${Object.values(AGENT_CATEGORIES).flat().length} DW Agents                           ║
╚═══════════════════════════════════════════════════════════╝
  `);
});