← back to Designer Wallcoverings

DW-Agents/dw-agents/agent-restart-analyzer/restart-analyzer-agent.ts

520 lines

#!/usr/bin/env tsx
/**
 * System Restart Analyzer Agent
 *
 * Analyzes PM2 process restart patterns to identify:
 * - Frequency and timing of restarts
 * - Processes with highest restart rates
 * - Common error patterns
 * - Recommendations for stability improvements
 *
 * Port: 7241
 */

import express from 'express';
import cookieParser from 'cookie-parser';
import Anthropic from '@anthropic-ai/sdk';
import { execSync } from 'child_process';
import fs from 'fs';
import path from 'path';

import { chatMiddleware } from '../shared-chat-integration';

const app = express();

// Global Authentication
app.use(cookieParser());
const PORT = process.env.PORT || 7241;

const anthropic = new Anthropic({
  apiKey: process.env.ANTHROPIC_API_KEY || ''
});

app.use(express.json());
app.use(express.static('public'));

// Add inter-agent chat widget
app.use(chatMiddleware({
  agentId: 'agent-restart',
  agentName: 'Restart Analyzer',
  port: 7241,
  category: 'agent'
}));


interface RestartData {
  processName: string;
  restartCount: number;
  uptime: number;
  status: string;
  memory: number;
  cpu: number;
  unstable_restarts: number;
}

interface LogError {
  timestamp: string;
  process: string;
  error: string;
  line: string;
}

// Get PM2 process information
function getPM2Processes(): RestartData[] {
  try {
    const output = execSync('pm2 jlist', { encoding: 'utf8' });
    const processes = JSON.parse(output);

    return processes.map((p: any) => ({
      processName: p.name || 'unknown',
      restartCount: p.pm2_env?.restart_time || 0,
      uptime: p.pm2_env?.pm_uptime || 0,
      status: p.pm2_env?.status || 'unknown',
      memory: p.monit?.memory || 0,
      cpu: p.monit?.cpu || 0,
      unstable_restarts: p.pm2_env?.unstable_restarts || 0
    }));
  } catch (error) {
    console.error('Error fetching PM2 data:', error);
    return [];
  }
}

// Analyze error logs
function analyzeErrorLogs(): LogError[] {
  const logsDir = '/root/Projects/Designer-Wallcoverings/DW-Agents/logs';
  const errors: LogError[] = [];

  try {
    const logFiles = fs.readdirSync(logsDir).filter(f => f.endsWith('-error.log'));

    for (const file of logFiles) {
      const filePath = path.join(logsDir, file);
      const processName = file.replace('-error.log', '');

      try {
        const content = fs.readFileSync(filePath, 'utf8');
        const lines = content.split('\n').filter(l => l.trim());

        // Get last 50 error lines
        const recentLines = lines.slice(-50);

        for (const line of recentLines) {
          if (line.toLowerCase().includes('error') ||
              line.toLowerCase().includes('exception') ||
              line.toLowerCase().includes('failed')) {

            // Try to extract timestamp
            const timestampMatch = line.match(/\d{4}-\d{2}-\d{2}|\d{2}:\d{2}:\d{2}/);

            errors.push({
              timestamp: timestampMatch ? timestampMatch[0] : 'unknown',
              process: processName,
              error: line.substring(0, 200), // First 200 chars
              line: line
            });
          }
        }
      } catch (err) {
        // Skip files that can't be read
      }
    }
  } catch (error) {
    console.error('Error reading log directory:', error);
  }

  return errors;
}

// Analyze restart patterns
async function analyzeRestartPatterns(): Promise<string> {
  console.log('📊 Analyzing system restart patterns...');

  const processes = getPM2Processes();
  const errors = analyzeErrorLogs();

  // Calculate statistics
  const totalRestarts = processes.reduce((sum, p) => sum + p.restartCount, 0);
  const avgRestarts = totalRestarts / processes.length;
  const maxRestarts = Math.max(...processes.map(p => p.restartCount));

  const problematicProcesses = processes
    .filter(p => p.restartCount > avgRestarts)
    .sort((a, b) => b.restartCount - a.restartCount);

  const stableProcesses = processes
    .filter(p => p.restartCount === 0)
    .length;

  // Group errors by process
  const errorsByProcess = errors.reduce((acc, err) => {
    if (!acc[err.process]) {
      acc[err.process] = [];
    }
    acc[err.process].push(err);
    return acc;
  }, {} as Record<string, LogError[]>);

  const analysisData = {
    summary: {
      totalProcesses: processes.length,
      totalRestarts,
      avgRestarts: avgRestarts.toFixed(2),
      maxRestarts,
      stableProcesses,
      problematicProcesses: problematicProcesses.length
    },
    processes: processes.map(p => ({
      name: p.processName,
      restarts: p.restartCount,
      status: p.status,
      uptime: Math.floor((Date.now() - p.uptime) / 1000 / 60), // minutes
      memory: (p.memory / 1024 / 1024).toFixed(2) + ' MB',
      unstableRestarts: p.unstable_restarts
    })),
    topProblematicProcesses: problematicProcesses.slice(0, 10).map(p => ({
      name: p.processName,
      restarts: p.restartCount,
      unstableRestarts: p.unstable_restarts
    })),
    errorSummary: Object.entries(errorsByProcess).map(([process, errs]) => ({
      process,
      errorCount: errs.length,
      recentErrors: errs.slice(-3).map(e => e.error)
    })),
    timestamp: new Date().toISOString()
  };

  // Use Claude to analyze patterns
  const prompt = `As a system reliability engineer, analyze the following PM2 process restart data and error logs:

${JSON.stringify(analysisData, null, 2)}

Provide a detailed analysis including:
1. **Critical Findings**: Identify the most concerning patterns
2. **Root Cause Analysis**: What's likely causing the restarts?
3. **Process-Specific Issues**: Break down problems by process
4. **Memory/Resource Patterns**: Analyze if resource constraints are a factor
5. **Stability Score**: Rate overall system stability (1-10)
6. **Immediate Actions**: What should be fixed first (prioritized list)
7. **Long-term Recommendations**: Preventive measures

Format the response in clear markdown with sections and bullet points.`;

  try {
    const message = await anthropic.messages.create({
      model: 'claude-opus-4-8',
      max_tokens: 4096,
      messages: [{
        role: 'user',
        content: prompt
      }]
    });

    const content = message.content[0];
    return content.type === 'text' ? content.text : 'Unable to analyze patterns';
  } catch (error) {
    console.error('Error with Claude analysis:', error);
    return generateFallbackAnalysis(analysisData);
  }
}

function generateFallbackAnalysis(data: any): string {
  return `# System Restart Pattern Analysis
## ${new Date().toLocaleString()}

### Summary Statistics
- **Total Processes:** ${data.summary.totalProcesses}
- **Total Restarts:** ${data.summary.totalRestarts}
- **Average Restarts per Process:** ${data.summary.avgRestarts}
- **Maximum Restarts (single process):** ${data.summary.maxRestarts}
- **Stable Processes (0 restarts):** ${data.summary.stableProcesses}
- **Problematic Processes:** ${data.summary.problematicProcesses}

### Top Problematic Processes
${data.topProblematicProcesses.map((p: any, i: number) =>
  `${i + 1}. **${p.name}**: ${p.restarts} restarts (${p.unstableRestarts} unstable)`
).join('\n')}

### Error Summary by Process
${data.errorSummary.slice(0, 5).map((e: any) =>
  `- **${e.process}**: ${e.errorCount} errors logged`
).join('\n')}

### Recommendations
1. Focus on processes with >20 restarts
2. Review error logs for common patterns
3. Consider increasing memory limits for high-restart processes
4. Implement health checks and auto-recovery
5. Set up alerting for restart spikes

---
*Analysis generated at ${new Date().toLocaleString()}*`;
}

// Dashboard HTML
function getDashboardHTML(analysis?: string): string {
  const processes = getPM2Processes();

  return `<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>Restart Analyzer Agent</title>
  <style>
    * { margin: 0; padding: 0; box-sizing: border-box; }
    body {
      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
      background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
      min-height: 100vh;
      padding: 20px;
    }
    .container {
      max-width: 1200px;
      margin: 0 auto;
    }
    .header {
      background: white;
      padding: 30px;
      border-radius: 15px;
      box-shadow: 0 10px 30px rgba(0,0,0,0.2);
      margin-bottom: 20px;
    }
    .header h1 {
      color: #667eea;
      font-size: 32px;
      margin-bottom: 10px;
    }
    .stats-grid {
      display: grid;
      grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
      gap: 15px;
      margin: 20px 0;
    }
    .stat-card {
      background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
      color: white;
      padding: 20px;
      border-radius: 10px;
      box-shadow: 0 5px 15px rgba(0,0,0,0.1);
    }
    .stat-card h3 {
      font-size: 14px;
      opacity: 0.9;
      margin-bottom: 10px;
    }
    .stat-card .value {
      font-size: 32px;
      font-weight: bold;
    }
    .process-list {
      background: white;
      padding: 20px;
      border-radius: 15px;
      box-shadow: 0 10px 30px rgba(0,0,0,0.2);
      margin-bottom: 20px;
    }
    .process-item {
      display: flex;
      justify-content: space-between;
      align-items: center;
      padding: 15px;
      border-bottom: 1px solid #eee;
    }
    .process-item:last-child {
      border-bottom: none;
    }
    .process-name {
      font-weight: 600;
      color: #333;
    }
    .restart-count {
      background: #ff6b6b;
      color: white;
      padding: 5px 15px;
      border-radius: 20px;
      font-weight: bold;
    }
    .restart-count.low {
      background: #51cf66;
    }
    .restart-count.medium {
      background: #ffd43b;
      color: #333;
    }
    .analysis-section {
      background: white;
      padding: 30px;
      border-radius: 15px;
      box-shadow: 0 10px 30px rgba(0,0,0,0.2);
      margin-bottom: 20px;
    }
    .btn {
      background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
      color: white;
      border: none;
      padding: 15px 30px;
      border-radius: 8px;
      font-size: 16px;
      font-weight: 600;
      cursor: pointer;
      box-shadow: 0 5px 15px rgba(0,0,0,0.2);
      transition: transform 0.2s;
    }
    .btn:hover {
      transform: translateY(-2px);
    }
    .markdown-content {
      line-height: 1.6;
    }
    .markdown-content h1, .markdown-content h2, .markdown-content h3 {
      margin: 20px 0 10px 0;
      color: #667eea;
    }
    .markdown-content ul {
      margin-left: 20px;
    }
    .markdown-content code {
      background: #f5f5f5;
      padding: 2px 5px;
      border-radius: 3px;
    }
  </style>
</head>
<body>
  <div class="container">
    <div class="header">
      <h1>🔄 System Restart Analyzer Agent</h1>
      <p>AI-powered analysis of PM2 process restart patterns</p>
    </div>

    <div class="stats-grid">
      <div class="stat-card">
        <h3>Total Processes</h3>
        <div class="value">${processes.length}</div>
      </div>
      <div class="stat-card">
        <h3>Total Restarts</h3>
        <div class="value">${processes.reduce((s, p) => s + p.restartCount, 0)}</div>
      </div>
      <div class="stat-card">
        <h3>Stable Processes</h3>
        <div class="value">${processes.filter(p => p.restartCount === 0).length}</div>
      </div>
      <div class="stat-card">
        <h3>Problematic</h3>
        <div class="value">${processes.filter(p => p.restartCount > 10).length}</div>
      </div>
    </div>

    <div class="process-list">
      <h2 style="margin-bottom: 20px; color: #667eea;">Process Restart Counts</h2>
      ${processes
        .sort((a, b) => b.restartCount - a.restartCount)
        .map(p => {
          let className = 'low';
          if (p.restartCount > 20) className = 'high';
          else if (p.restartCount > 5) className = 'medium';

          return `
            <div class="process-item">
              <span class="process-name">${p.processName}</span>
              <span class="restart-count ${className}">${p.restartCount} restarts</span>
            </div>
          `;
        }).join('')}
    </div>

    <div style="text-align: center; margin: 30px 0;">
      <button class="btn" onclick="runAnalysis()">
        🤖 Run AI Analysis
      </button>
    </div>

    ${analysis ? `
      <div class="analysis-section">
        <h2 style="margin-bottom: 20px; color: #667eea;">AI Analysis Results</h2>
        <div class="markdown-content">
          ${analysis.replace(/\n/g, '<br>').replace(/#{1,3}\s/g, '<h3>').replace(/<h3>/g, '</h3><h3>').replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>')}
        </div>
      </div>
    ` : ''}
  </div>

  <script>
    async function runAnalysis() {
      const btn = event.target;
      btn.disabled = true;
      btn.textContent = '🔄 Analyzing...';

      try {
        const response = await fetch('/api/analyze', {
          method: 'POST'
        });

        if (response.ok) {
          location.reload();
        }
      } catch (error) {
        alert('Analysis failed: ' + error.message);
      } finally {
        btn.disabled = false;
        btn.textContent = '🤖 Run AI Analysis';
      }
    }
  </script>
</body>
</html>`;
}

// Routes
app.get('/', async (req, res) => {
  res.send(getDashboardHTML());
});

app.post('/api/analyze', async (req, res) => {
  try {
    const analysis = await analyzeRestartPatterns();

    // Save analysis to file
    const reportDate = new Date().toISOString().split('T')[0];
    const reportPath = `/root/Projects/Designer-Wallcoverings/DW-Agents/reports/restart-analysis-${reportDate}.md`;
    fs.writeFileSync(reportPath, analysis);

    res.json({
      success: true,
      analysis,
      reportPath
    });
  } catch (error) {
    res.status(500).json({
      success: false,
      error: error instanceof Error ? error.message : 'Unknown error'
    });
  }
});

app.get('/api/processes', (req, res) => {
  res.json(getPM2Processes());
});


// Metrics endpoint for daily reporting
app.get("/api/metrics", (req: Request, res: Response) => {
  res.json({
    status: "online",
    uptime: process.uptime(),
    responseTime: 0,
    tasksCompleted: 0,
    errorsToday: 0,
    lastActivity: new Date().toISOString(),
    qnaReadiness: 100
  });
});

app.listen(PORT, () => {
  console.log(`🔄 Restart Analyzer Agent running on port ${PORT}`);
  console.log(`📊 Dashboard: http://localhost:${PORT}`);
});