`DW-Agents/dw-agents/agent-log-monitor/log-monitor-agent.ts.backup-auth-1762615744`

685 lines
/**
 * DW-Agents: Log Monitor Agent
 *
 * Automated log analysis and error detection:
 * - Continuous monitoring of all PM2 logs
 * - Error pattern detection
 * - Automatic task creation for issues
 * - Alert escalation
 * - Real-time dashboard
 *
 * Port: 7239
 */

import Anthropic from '@anthropic-ai/sdk';
import express, { Request, Response } from 'express';
import session from 'express-session';
import * as fs from 'fs';
import * as path from 'path';
import { exec } from 'child_process';
import { promisify } from 'util';
import { requireAuth as ssoAuth, handleLogin, SSO_TOKEN_NAME, SSO_TOKEN_VALUE } from '../shared-auth';
import cookieParser from 'cookie-parser';
const { getUniversalHeader } = require('../shared-ui-components.js');

const execAsync = promisify(exec);

const app = express();
const PORT = 7239;

// Paths
const LOGS_DIR = '/root/DW-Agents/logs';
const CLAUDE_DEBUG_DIR = '/root/.claude/debug';
const ISSUES_FILE = path.join(__dirname, 'detected-issues.json');
const TASKS_FILE = path.join(__dirname, '../tasks-database.json');

// Anthropic Claude
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || '';
const anthropic = new Anthropic({ apiKey: ANTHROPIC_API_KEY });

// Error patterns to detect
const ERROR_PATTERNS = [
  { pattern: /error|exception|fatal|crash|failed/i, severity: 'high', category: 'error' },
  { pattern: /warn|warning/i, severity: 'medium', category: 'warning' },
  { pattern: /ECONNREFUSED|ETIMEDOUT|ENOTFOUND/i, severity: 'high', category: 'network' },
  { pattern: /Cannot find module|MODULE_NOT_FOUND/i, severity: 'critical', category: 'dependency' },
  { pattern: /Port \d+ is already in use/i, severity: 'critical', category: 'port_conflict' },
  { pattern: /EADDRINUSE/i, severity: 'critical', category: 'port_conflict' },
  { pattern: /Memory usage|heap out of memory/i, severity: 'critical', category: 'memory' },
  { pattern: /401|403|500|502|503|504/i, severity: 'high', category: 'http_error' },
  { pattern: /restart/i, severity: 'medium', category: 'restart' },
  { pattern: /Unhandled|UnhandledPromiseRejection/i, severity: 'high', category: 'unhandled' },
];

// Middleware
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(cookieParser());

declare module 'express-session' {
  interface SessionData {
    authenticated: boolean;
  }
}

app.use(
  session({
    secret: 'dw-agents-log-monitor-2025',
    resave: false,
    saveUninitialized: true,
    rolling: true,
    cookie: {
      secure: false,
      httpOnly: true,
      maxAge: 7 * 24 * 60 * 60 * 1000,
    },
  })
);

// Authentication
app.post('/login', handleLogin);

const requireAuth = (req: Request, res: Response, next: any) => {
  if (req.cookies[SSO_TOKEN_NAME] === SSO_TOKEN_VALUE || req.session.authenticated) {
    next();
  } else {
    res.redirect('/login');
  }
};

app.get('/login', (req: Request, res: Response) => {
  res.send(`
    <!DOCTYPE html>
    <html>
    <head>
      <title>Log Monitor Agent - Login</title>
      <style>
        body {
          font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
          background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
          display: flex;
          justify-content: center;
          align-items: center;
          height: 100vh;
          margin: 0;
        }
        .login-box {
          background: white;
          padding: 40px;
          border-radius: 10px;
          box-shadow: 0 10px 40px rgba(0,0,0,0.3);
          width: 300px;
        }
        h2 { margin-top: 0; color: #333; }
        input {
          width: 100%;
          padding: 12px;
          margin: 10px 0;
          border: 1px solid #ddd;
          border-radius: 5px;
          box-sizing: border-box;
        }
        button {
          width: 100%;
          padding: 12px;
          background: #f5576c;
          color: white;
          border: none;
          border-radius: 5px;
          cursor: pointer;
          font-size: 16px;
        }
        button:hover { background: #e04555; }
      </style>
    </head>
    <body>
      <div class="login-box">
        <h2>📋 Log Monitor Agent</h2>
        <form method="POST" action="/login">
          <input type="password" name="password" placeholder="Password" required autofocus />
          <button type="submit">Login</button>
        </form>
      </div>
    </body>
    </html>
  `);
});

// Store detected issues
interface DetectedIssue {
  id: string;
  timestamp: string;
  agent: string;
  severity: 'critical' | 'high' | 'medium' | 'low';
  category: string;
  message: string;
  logFile: string;
  taskCreated: boolean;
  taskId?: string;
  resolved: boolean;
}

let detectedIssues: DetectedIssue[] = [];

// Load existing issues
function loadIssues() {
  if (fs.existsSync(ISSUES_FILE)) {
    try {
      detectedIssues = JSON.parse(fs.readFileSync(ISSUES_FILE, 'utf-8'));
    } catch (e) {
      detectedIssues = [];
    }
  }
}

function saveIssues() {
  fs.writeFileSync(ISSUES_FILE, JSON.stringify(detectedIssues, null, 2));
}

loadIssues();

// Analyze log entry
function analyzeLogEntry(logLine: string, agentName: string, logFile: string): DetectedIssue | null {
  for (const pattern of ERROR_PATTERNS) {
    if (pattern.pattern.test(logLine)) {
      return {
        id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
        timestamp: new Date().toISOString(),
        agent: agentName,
        severity: pattern.severity as any,
        category: pattern.category,
        message: logLine.substring(0, 500),
        logFile,
        taskCreated: false,
        resolved: false,
      };
    }
  }
  return null;
}

// Create task in orchestrator
async function createTask(issue: DetectedIssue) {
  const task = {
    id: `task-${issue.id}`,
    title: `[${issue.severity.toUpperCase()}] ${issue.category} in ${issue.agent}`,
    description: `Automated detection from log monitor:\n\n${issue.message}\n\nLog: ${issue.logFile}`,
    status: 'pending',
    priority: issue.severity === 'critical' ? 'critical' : issue.severity === 'high' ? 'high' : 'medium',
    assignedAgent: determineAgent(issue),
    createdAt: new Date().toISOString(),
    autoGenerated: true,
    source: 'log-monitor',
    issueId: issue.id,
  };

  // Save to tasks database
  let tasks: any[] = [];
  if (fs.existsSync(TASKS_FILE)) {
    try {
      const data = JSON.parse(fs.readFileSync(TASKS_FILE, 'utf-8'));
      tasks = data.tasks || [];
    } catch (e) {
      tasks = [];
    }
  }

  tasks.push(task);
  fs.writeFileSync(TASKS_FILE, JSON.stringify({ tasks }, null, 2));

  issue.taskCreated = true;
  issue.taskId = task.id;

  console.log(`✅ Created task ${task.id} for ${issue.agent} - ${issue.category}`);
  return task.id;
}

// Determine which agent should handle the issue
function determineAgent(issue: DetectedIssue): string {
  if (issue.severity === 'critical') return 'needs-attention';
  if (issue.category === 'port_conflict' || issue.category === 'memory') return 'server-uptime';
  if (issue.agent === 'claude-code') return 'task-orchestrator'; // Claude Code errors go to orchestrator
  if (issue.agent.includes('shopify')) return 'shopify-store';
  if (issue.agent.includes('purchasing')) return 'purchasing-office';
  if (issue.agent.includes('marketing')) return 'marketing';
  if (issue.agent.includes('zendesk')) return 'zendesk-chat';
  if (issue.agent.includes('accounting')) return 'accounting';
  if (issue.agent.includes('legal')) return 'legal-team';

  return 'task-orchestrator'; // Default
}

// Monitor logs continuously
async function monitorLogs() {
  try {
    // Monitor DW-Agents logs
    const logFiles = fs.readdirSync(LOGS_DIR).filter(f => f.endsWith('.log'));

    for (const logFile of logFiles) {
      const filePath = path.join(LOGS_DIR, logFile);
      const agentName = logFile.replace('-error.log', '').replace('-out.log', '');

      // Read last 100 lines
      try {
        const { stdout } = await execAsync(`tail -n 100 "${filePath}"`);
        const lines = stdout.split('\n').filter(l => l.trim());

        for (const line of lines) {
          const issue = analyzeLogEntry(line, agentName, logFile);

          if (issue) {
            // Check if we already detected this recently (avoid duplicates)
            const recentDuplicate = detectedIssues.find(
              i => i.agent === issue.agent &&
                   i.category === issue.category &&
                   i.message === issue.message &&
                   Date.now() - new Date(i.timestamp).getTime() < 5 * 60 * 1000 // 5 min
            );

            if (!recentDuplicate) {
              detectedIssues.push(issue);
              console.log(`🚨 Detected ${issue.severity} issue in ${issue.agent}: ${issue.category}`);

              // Create task for high and critical issues
              if (issue.severity === 'high' || issue.severity === 'critical') {
                await createTask(issue);
              }
            }
          }
        }
      } catch (error) {
        // Skip if file doesn't exist or can't be read
      }
    }

    // Monitor Claude Code debug logs
    try {
      if (fs.existsSync(CLAUDE_DEBUG_DIR)) {
        const claudeDebugFiles = fs.readdirSync(CLAUDE_DEBUG_DIR)
          .filter(f => f.endsWith('.txt'))
          .sort((a, b) => {
            const statA = fs.statSync(path.join(CLAUDE_DEBUG_DIR, a));
            const statB = fs.statSync(path.join(CLAUDE_DEBUG_DIR, b));
            return statB.mtimeMs - statA.mtimeMs;
          })
          .slice(0, 10); // Check only 10 most recent files

        for (const debugFile of claudeDebugFiles) {
          const filePath = path.join(CLAUDE_DEBUG_DIR, debugFile);
          const stats = fs.statSync(filePath);

          // Only check files modified in last hour
          if (Date.now() - stats.mtimeMs < 60 * 60 * 1000) {
            try {
              const { stdout } = await execAsync(`tail -n 200 "${filePath}"`);
              const lines = stdout.split('\n').filter(l => l.trim());

              for (const line of lines) {
                const issue = analyzeLogEntry(line, 'claude-code', `debug/${debugFile}`);

                if (issue) {
                  const recentDuplicate = detectedIssues.find(
                    i => i.agent === 'claude-code' &&
                         i.category === issue.category &&
                         i.message === issue.message &&
                         Date.now() - new Date(i.timestamp).getTime() < 5 * 60 * 1000
                  );

                  if (!recentDuplicate) {
                    detectedIssues.push(issue);
                    console.log(`🚨 Detected ${issue.severity} Claude Code issue: ${issue.category}`);

                    // Create task for high and critical issues
                    if (issue.severity === 'high' || issue.severity === 'critical') {
                      await createTask(issue);
                    }
                  }
                }
              }
            } catch (error) {
              // Skip if file can't be read
            }
          }
        }
      }
    } catch (error) {
      // Claude debug dir might not exist
    }

    // Keep only last 1000 issues
    if (detectedIssues.length > 1000) {
      detectedIssues = detectedIssues.slice(-1000);
    }

    saveIssues();
  } catch (error) {
    console.error('Error monitoring logs:', error);
  }
}

// Check PM2 process status
async function checkPM2Status() {
  try {
    const { stdout } = await execAsync('pm2 jlist');
    const processes = JSON.parse(stdout);

    for (const proc of processes) {
      // Check for frequent restarts
      if (proc.pm2_env.restart_time > 5 && proc.pm2_env.uptime < 60000) {
        const issue: DetectedIssue = {
          id: `pm2-${Date.now()}`,
          timestamp: new Date().toISOString(),
          agent: proc.name,
          severity: 'critical',
          category: 'frequent_restarts',
          message: `Process ${proc.name} has restarted ${proc.pm2_env.restart_time} times recently`,
          logFile: 'pm2',
          taskCreated: false,
          resolved: false,
        };

        detectedIssues.push(issue);
        await createTask(issue);
      }

      // Check memory usage
      if (proc.monit && proc.monit.memory > 400 * 1024 * 1024) { // > 400MB
        const issue: DetectedIssue = {
          id: `mem-${Date.now()}`,
          timestamp: new Date().toISOString(),
          agent: proc.name,
          severity: 'high',
          category: 'high_memory',
          message: `Process ${proc.name} using ${Math.round(proc.monit.memory / 1024 / 1024)}MB memory`,
          logFile: 'pm2',
          taskCreated: false,
          resolved: false,
        };

        const recentDuplicate = detectedIssues.find(
          i => i.agent === issue.agent && i.category === 'high_memory' &&
               Date.now() - new Date(i.timestamp).getTime() < 10 * 60 * 1000
        );

        if (!recentDuplicate) {
          detectedIssues.push(issue);
          await createTask(issue);
        }
      }
    }

    saveIssues();
  } catch (error) {
    console.error('Error checking PM2 status:', error);
  }
}

// Start monitoring interval
setInterval(monitorLogs, 30000); // Every 30 seconds
setInterval(checkPM2Status, 60000); // Every 1 minute

// Initial scan
monitorLogs();
checkPM2Status();

// Dashboard
app.get('/', requireAuth, (req: Request, res: Response) => {
  const criticalCount = detectedIssues.filter(i => !i.resolved && i.severity === 'critical').length;
  const highCount = detectedIssues.filter(i => !i.resolved && i.severity === 'high').length;
  const mediumCount = detectedIssues.filter(i => !i.resolved && i.severity === 'medium').length;

  res.send(`
    <!DOCTYPE html>
    <html>
    <head>
      <title>Log Monitor Agent - Dashboard</title>
      <meta http-equiv="refresh" content="30">
      <style>
        * { margin: 0; padding: 0; box-sizing: border-box; }
        body {
          font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
          background: #f5f5f5;
        }
        .header {
          background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
          color: white;
          padding: 20px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
        }
        .header h1 { font-size: 24px; margin-bottom: 10px; }
        .header p { font-size: 14px; opacity: 0.9; }
        .stats {
          display: flex;
          gap: 20px;
          padding: 20px;
          flex-wrap: wrap;
        }
        .stat-box {
          background: white;
          padding: 20px;
          border-radius: 10px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
          min-width: 150px;
          text-align: center;
        }
        .stat-box h3 {
          font-size: 14px;
          color: #666;
          margin-bottom: 10px;
        }
        .stat-box p {
          font-size: 32px;
          font-weight: bold;
        }
        .stat-box.critical p { color: #e74c3c; }
        .stat-box.high p { color: #e67e22; }
        .stat-box.medium p { color: #f39c12; }
        .stat-box.total p { color: #3498db; }
        .content {
          margin: 20px;
          background: white;
          padding: 20px;
          border-radius: 10px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
        }
        table {
          width: 100%;
          border-collapse: collapse;
          margin-top: 15px;
        }
        th, td {
          padding: 12px;
          text-align: left;
          border-bottom: 1px solid #eee;
          font-size: 13px;
        }
        th {
          background: #f9f9f9;
          font-weight: 600;
        }
        tr:hover { background: #fafafa; }
        .severity {
          padding: 4px 12px;
          border-radius: 12px;
          font-size: 11px;
          font-weight: bold;
          text-transform: uppercase;
        }
        .severity.critical {
          background: #fee;
          color: #e74c3c;
        }
        .severity.high {
          background: #fef3e0;
          color: #e67e22;
        }
        .severity.medium {
          background: #fef9e0;
          color: #f39c12;
        }
        .severity.low {
          background: #e8f5e9;
          color: #27ae60;
        }
        .badge {
          display: inline-block;
          padding: 4px 8px;
          border-radius: 4px;
          font-size: 11px;
          background: #3498db;
          color: white;
        }
        .badge.task { background: #27ae60; }
        .badge.resolved { background: #95a5a6; }
        .btn {
          padding: 6px 12px;
          background: #3498db;
          color: white;
          border: none;
          border-radius: 4px;
          cursor: pointer;
          font-size: 12px;
        }
        .btn:hover { background: #2980b9; }
        .btn.resolve { background: #27ae60; }
        .btn.resolve:hover { background: #229954; }
      </style>
    </head>
    <body style="padding: 20px;">
      ${getUniversalHeader('Log Monitor', PORT, 5)}

      <div class="header" style="margin-top: 20px;">
        <h1>📋 Log Monitor Agent</h1>
        <p>Real-time log analysis and automated issue detection</p>
        <p>Monitoring: DW-Agents logs (/root/DW-Agents/logs) + Claude Code debug logs (/root/.claude/debug)</p>
      </div>

      <div class="stats">
        <div class="stat-box critical">
          <h3>Critical Issues</h3>
          <p>${criticalCount}</p>
        </div>
        <div class="stat-box high">
          <h3>High Priority</h3>
          <p>${highCount}</p>
        </div>
        <div class="stat-box medium">
          <h3>Medium Priority</h3>
          <p>${mediumCount}</p>
        </div>
        <div class="stat-box total">
          <h3>Total Detected</h3>
          <p>${detectedIssues.length}</p>
        </div>
      </div>

      <div class="content">
        <h2>Recent Issues</h2>
        <table>
          <tr>
            <th>Time</th>
            <th>Agent</th>
            <th>Severity</th>
            <th>Category</th>
            <th>Message</th>
            <th>Status</th>
            <th>Actions</th>
          </tr>
          ${detectedIssues.slice(-50).reverse().map(issue => `
            <tr>
              <td>${new Date(issue.timestamp).toLocaleString()}</td>
              <td><strong>${issue.agent}</strong></td>
              <td><span class="severity ${issue.severity}">${issue.severity}</span></td>
              <td>${issue.category}</td>
              <td style="max-width: 400px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">${issue.message}</td>
              <td>
                ${issue.resolved ? '<span class="badge resolved">Resolved</span>' : ''}
                ${issue.taskCreated ? `<span class="badge task">Task ${issue.taskId}</span>` : ''}
              </td>
              <td>
                ${!issue.resolved ? `<button class="btn resolve" onclick="resolveIssue('${issue.id}')">Mark Resolved</button>` : ''}
                ${!issue.taskCreated ? `<button class="btn" onclick="createTask('${issue.id}')">Create Task</button>` : ''}
              </td>
            </tr>
          `).join('')}
        </table>
      </div>

      <script>
        function resolveIssue(id) {
          fetch('/api/resolve/' + id, { method: 'POST' })
            .then(() => location.reload());
        }

        function createTask(id) {
          fetch('/api/create-task/' + id, { method: 'POST' })
            .then(() => location.reload());
        }

        // Auto-refresh every 30 seconds
        setTimeout(() => location.reload(), 30000);
      </script>
    </body>
    </html>
  `);
});

// API: Get issues
app.get('/api/issues', requireAuth, (req: Request, res: Response) => {
  res.json({ issues: detectedIssues });
});

// API: Resolve issue
app.post('/api/resolve/:id', requireAuth, (req: Request, res: Response) => {
  const issue = detectedIssues.find(i => i.id === req.params.id);
  if (issue) {
    issue.resolved = true;
    saveIssues();
  }
  res.json({ success: true });
});

// API: Create task manually
app.post('/api/create-task/:id', requireAuth, async (req: Request, res: Response) => {
  const issue = detectedIssues.find(i => i.id === req.params.id);
  if (issue && !issue.taskCreated) {
    await createTask(issue);
    saveIssues();
    res.json({ success: true, taskId: issue.taskId });
  } else {
    res.json({ success: false });
  }
});

// Health check
app.get('/health', (req: Request, res: Response) => {
  res.json({
    status: 'healthy',
    agent: 'log-monitor',
    port: PORT,
    issuesDetected: detectedIssues.length,
    unresolvedCritical: detectedIssues.filter(i => !i.resolved && i.severity === 'critical').length,
    timestamp: new Date().toISOString()
  });
});

// Start server
app.listen(PORT, () => {
  console.log(`📋 Log Monitor Agent running on port ${PORT}`);
  console.log(`📊 Dashboard: http://localhost:${PORT}`);
  console.log(`🔍 Monitoring ${LOGS_DIR}`);
});

// Graceful shutdown
process.on('SIGTERM', () => {
  console.log('SIGTERM received, shutting down gracefully...');
  saveIssues();
  process.exit(0);
});

process.on('SIGINT', () => {
  console.log('SIGINT received, shutting down gracefully...');
  saveIssues();
  process.exit(0);
});