← back to Designer Wallcoverings

DW-Agents/dw-agents/agent-monitor-dashboard/monitor-dashboard.js

318 lines

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
    return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const express_1 = __importDefault(require("express"));
const child_process_1 = require("child_process");
const util_1 = require("util");
const promises_1 = __importDefault(require("fs/promises"));
const path_1 = __importDefault(require("path"));
const execAsync = (0, util_1.promisify)(child_process_1.exec);
const app = (0, express_1.default)();
const PORT = 9899;
// Middleware
app.use(express_1.default.json());
app.use(express_1.default.static('public'));
// Agent categories for grouping
const AGENT_CATEGORIES = {
    'Executive': ['dw-executive-ceo', 'dw-executive-cfo', 'dw-executive-coo', 'dw-executive-vp-ops'],
    'Operations': ['dw-master-hub', 'dw-control-panel', 'dw-dashboard-monitor', 'dw-crash-loop-resolver'],
    'Customer Service': ['dw-zendesk-chat', 'dw-shopify-store', 'dw-digital-samples', 'dw-needs-attention'],
    'Business Intelligence': ['dw-bigquery-gov', 'dw-trend-research', 'dw-marketing', 'dw-accounting'],
    'Infrastructure': ['dw-server-uptime', 'dw-log-monitor', 'dw-ui-manager', 'dw-legal-team'],
    'Support': ['dw-purchasing-office']
};
// Health thresholds
const THRESHOLDS = {
    RESTART_WARNING: 10,
    RESTART_CRITICAL: 20,
    RESTART_SEVERE: 50,
    MEMORY_WARNING: 200, // MB
    MEMORY_CRITICAL: 400, // MB
    CPU_WARNING: 50, // %
    CPU_CRITICAL: 80, // %
    UPTIME_GOOD: 3600, // 1 hour
    UPTIME_WARNING: 300 // 5 minutes
};
// Get PM2 stats
async function getPM2Stats() {
    try {
        const { stdout } = await execAsync('pm2 jlist');
        const processes = JSON.parse(stdout);
        return processes.map((proc) => {
            const uptime = proc.pm2_env.pm_uptime ? Date.now() - proc.pm2_env.pm_uptime : 0;
            const uptimeSeconds = uptime / 1000;
            return {
                name: proc.name,
                status: proc.pm2_env.status,
                restarts: proc.pm2_env.restart_time || 0,
                cpu: proc.monit?.cpu || 0,
                memory: proc.monit?.memory ? Math.round(proc.monit.memory / 1024 / 1024) : 0, // MB
                uptime: uptimeSeconds,
                uptimeFormatted: formatUptime(uptimeSeconds),
                pm_id: proc.pm_id,
                error_count: proc.pm2_env.unstable_restarts || 0,
                created_at: proc.pm2_env.created_at,
                instances: proc.pm2_env.instances || 1,
                exec_mode: proc.pm2_env.exec_mode
            };
        }).filter((proc) => proc.name.startsWith('dw-')); // Only DW agents
    }
    catch (error) {
        console.error('Error getting PM2 stats:', error);
        return [];
    }
}
// Get latest error logs
async function getErrorLogs(agentName, lines = 5) {
    try {
        const logPath = `/root/DW-Agents/logs/${agentName.replace('dw-', '')}-error.log`;
        const fileExists = await promises_1.default.access(logPath).then(() => true).catch(() => false);
        if (!fileExists) {
            return [];
        }
        const { stdout } = await execAsync(`tail -n ${lines} "${logPath}" 2>/dev/null || echo ""`);
        return stdout.split('\n').filter(line => line.trim()).slice(-lines);
    }
    catch (error) {
        return [];
    }
}
// Format uptime
function formatUptime(seconds) {
    if (seconds < 60)
        return `${Math.floor(seconds)}s`;
    if (seconds < 3600)
        return `${Math.floor(seconds / 60)}m ${Math.floor(seconds % 60)}s`;
    if (seconds < 86400)
        return `${Math.floor(seconds / 3600)}h ${Math.floor((seconds % 3600) / 60)}m`;
    return `${Math.floor(seconds / 86400)}d ${Math.floor((seconds % 86400) / 3600)}h`;
}
// Calculate health score
function calculateHealthScore(stats) {
    let totalScore = 0;
    let agentCount = 0;
    for (const agent of stats) {
        let agentScore = 100;
        // Status check
        if (agent.status !== 'online') {
            agentScore -= 50;
        }
        // Restart penalty
        if (agent.restarts > THRESHOLDS.RESTART_SEVERE) {
            agentScore -= 30;
        }
        else if (agent.restarts > THRESHOLDS.RESTART_CRITICAL) {
            agentScore -= 20;
        }
        else if (agent.restarts > THRESHOLDS.RESTART_WARNING) {
            agentScore -= 10;
        }
        // Memory penalty
        if (agent.memory > THRESHOLDS.MEMORY_CRITICAL) {
            agentScore -= 15;
        }
        else if (agent.memory > THRESHOLDS.MEMORY_WARNING) {
            agentScore -= 5;
        }
        // CPU penalty
        if (agent.cpu > THRESHOLDS.CPU_CRITICAL) {
            agentScore -= 15;
        }
        else if (agent.cpu > THRESHOLDS.CPU_WARNING) {
            agentScore -= 5;
        }
        // Uptime bonus
        if (agent.uptime > THRESHOLDS.UPTIME_GOOD) {
            agentScore = Math.min(100, agentScore + 10);
        }
        else if (agent.uptime < THRESHOLDS.UPTIME_WARNING) {
            agentScore -= 10;
        }
        totalScore += Math.max(0, agentScore);
        agentCount++;
    }
    return agentCount > 0 ? Math.round(totalScore / agentCount) : 0;
}
// API Endpoints
// Get all agent stats
app.get('/api/stats', async (req, res) => {
    try {
        const stats = await getPM2Stats();
        const healthScore = calculateHealthScore(stats);
        // Get error logs for problematic agents
        for (const agent of stats) {
            if (agent.restarts > THRESHOLDS.RESTART_WARNING || agent.status !== 'online') {
                agent.errorLogs = await getErrorLogs(agent.name, 3);
            }
        }
        // Categorize agents
        const categorizedStats = {};
        for (const [category, agentNames] of Object.entries(AGENT_CATEGORIES)) {
            categorizedStats[category] = stats.filter((s) => agentNames.includes(s.name));
        }
        // Find uncategorized agents
        const allCategorizedNames = Object.values(AGENT_CATEGORIES).flat();
        const uncategorized = stats.filter((s) => !allCategorizedNames.includes(s.name));
        if (uncategorized.length > 0) {
            categorizedStats['Other'] = uncategorized;
        }
        res.json({
            success: true,
            data: {
                stats,
                categorized: categorizedStats,
                healthScore,
                timestamp: new Date().toISOString(),
                thresholds: THRESHOLDS,
                summary: {
                    total: stats.length,
                    online: stats.filter((s) => s.status === 'online').length,
                    errored: stats.filter((s) => s.status === 'errored').length,
                    stopped: stats.filter((s) => s.status === 'stopped').length,
                    launching: stats.filter((s) => s.status === 'launching').length,
                    criticalRestarts: stats.filter((s) => s.restarts > THRESHOLDS.RESTART_CRITICAL).length,
                    highMemory: stats.filter((s) => s.memory > THRESHOLDS.MEMORY_WARNING).length,
                    highCpu: stats.filter((s) => s.cpu > THRESHOLDS.CPU_WARNING).length
                }
            }
        });
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// Restart agent
app.post('/api/restart/:name', async (req, res) => {
    try {
        const { name } = req.params;
        const { stdout } = await execAsync(`pm2 restart ${name}`);
        res.json({ success: true, message: `Agent ${name} restarted`, output: stdout });
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// Stop agent
app.post('/api/stop/:name', async (req, res) => {
    try {
        const { name } = req.params;
        const { stdout } = await execAsync(`pm2 stop ${name}`);
        res.json({ success: true, message: `Agent ${name} stopped`, output: stdout });
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// Delete agent
app.post('/api/delete/:name', async (req, res) => {
    try {
        const { name } = req.params;
        const { stdout } = await execAsync(`pm2 delete ${name}`);
        res.json({ success: true, message: `Agent ${name} deleted`, output: stdout });
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// View full logs
app.get('/api/logs/:name', async (req, res) => {
    try {
        const { name } = req.params;
        const { lines = 100 } = req.query;
        const errorLogs = await getErrorLogs(name, Number(lines));
        const { stdout: outputLogs } = await execAsync(`pm2 logs ${name} --nostream --lines ${lines} 2>/dev/null || echo ""`);
        res.json({
            success: true,
            data: {
                errorLogs,
                outputLogs: outputLogs.split('\n').filter(line => line.trim()),
                agent: name
            }
        });
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// Export health report
app.get('/api/export', async (req, res) => {
    try {
        const stats = await getPM2Stats();
        const healthScore = calculateHealthScore(stats);
        const timestamp = new Date().toISOString();
        const report = {
            timestamp,
            healthScore,
            summary: {
                total: stats.length,
                online: stats.filter((s) => s.status === 'online').length,
                errored: stats.filter((s) => s.status === 'errored').length,
                stopped: stats.filter((s) => s.status === 'stopped').length,
                criticalRestarts: stats.filter((s) => s.restarts > THRESHOLDS.RESTART_CRITICAL).length
            },
            agents: stats.map((s) => ({
                name: s.name,
                status: s.status,
                restarts: s.restarts,
                cpu: s.cpu,
                memory: s.memory,
                uptime: s.uptimeFormatted,
                issues: []
                    .concat(s.restarts > THRESHOLDS.RESTART_CRITICAL ? [`High restarts: ${s.restarts}`] : [])
                    .concat(s.memory > THRESHOLDS.MEMORY_WARNING ? [`High memory: ${s.memory}MB`] : [])
                    .concat(s.cpu > THRESHOLDS.CPU_WARNING ? [`High CPU: ${s.cpu}%`] : [])
                    .concat(s.status !== 'online' ? [`Status: ${s.status}`] : [])
            }))
        };
        res.setHeader('Content-Type', 'application/json');
        res.setHeader('Content-Disposition', `attachment; filename="dw-agents-health-report-${Date.now()}.json"`);
        res.send(JSON.stringify(report, null, 2));
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// Batch restart problematic agents
app.post('/api/restart-problematic', async (req, res) => {
    try {
        const stats = await getPM2Stats();
        const problematic = stats.filter((s) => s.restarts > THRESHOLDS.RESTART_CRITICAL ||
            s.status === 'errored' ||
            s.status === 'launching');
        const results = [];
        for (const agent of problematic) {
            try {
                await execAsync(`pm2 restart ${agent.name}`);
                results.push({ agent: agent.name, success: true });
            }
            catch (error) {
                results.push({ agent: agent.name, success: false, error: String(error) });
            }
        }
        res.json({ success: true, restarted: results });
    }
    catch (error) {
        res.status(500).json({ success: false, error: error.message });
    }
});
// Serve dashboard HTML
app.get('/', (req, res) => {
    res.sendFile(path_1.default.join(__dirname, 'public', 'index.html'));
});
// Start server
app.listen(PORT, () => {
    console.log(`
╔═══════════════════════════════════════════════════════════╗
║         DW-AGENTS MONITOR DASHBOARD                       ║
║         Real-time Health Monitoring System                ║
╠═══════════════════════════════════════════════════════════╣
║  🚀 Server running on port ${PORT}                          ║
║  📊 Dashboard URL: http://45.61.58.125:${PORT}              ║
║  🔄 Auto-refresh: Every 10 seconds                        ║
║  ⚠️  Monitoring ${Object.values(AGENT_CATEGORIES).flat().length} DW Agents                           ║
╚═══════════════════════════════════════════════════════════╝
  `);
});