← back to Designer Wallcoverings

DW-Agents/dw-agents/agent-log-monitor/log-monitor-agent.ts

1202 lines

import cookieParser from "cookie-parser";
/**
 * DW-Agents: Log Monitor Agent
 *
 * Automated log analysis and error detection:
 * - Continuous monitoring of all PM2 logs
 * - Error pattern detection
 * - Automatic task creation for issues
 * - Alert escalation
 * - Real-time dashboard
 *
 * Port: 7239
 */

import Anthropic from '@anthropic-ai/sdk';
import express, { Request, Response } from 'express';
import { requireGlobalAuth } from '../shared-global-auth';
import session from 'express-session';
import * as fs from 'fs';
import * as path from 'path';
import { exec } from 'child_process';
import { promisify } from 'util';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
import { requireAuth as ssoAuth, handleLogin, SSO_TOKEN_NAME, SSO_TOKEN_VALUE } from '../shared-auth';
// cookieParser already imported at top

import { chatMiddleware } from '../shared-chat-integration';
import { getUniversalHeader, getThemeStyles } from '../shared-ui-components.js';

// Fix __dirname for ES modules
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const execAsync = promisify(exec);

const app = express();

// Global Authentication
const PORT = 7239;

// Global Authentication - MUST come before any other middleware
app.use(cookieParser());

// Add inter-agent chat widget (after auth)
// TEMPORARILY DISABLED for debugging
// app.use(chatMiddleware({
//   agentId: 'agent-log-monitor',
//   agentName: 'Log Monitor',
//   port: 7239,
//   category: 'agent'
// }));


// Paths
const LOGS_DIR = '/root/Projects/Designer-Wallcoverings/DW-Agents/logs';
const CLAUDE_DEBUG_DIR = '/root/.claude/debug';
const ISSUES_FILE = path.join(__dirname, 'detected-issues.json');
const TASKS_FILE = path.join(__dirname, '../tasks-database.json');

// Anthropic Claude
const ANTHROPIC_API_KEY = process.env.ANTHROPIC_API_KEY || '';
const anthropic = new Anthropic({ apiKey: ANTHROPIC_API_KEY });

// Error patterns to detect
const ERROR_PATTERNS = [
  { pattern: /error|exception|fatal|crash|failed/i, severity: 'high', category: 'error' },
  { pattern: /warn|warning/i, severity: 'medium', category: 'warning' },
  { pattern: /ECONNREFUSED|ETIMEDOUT|ENOTFOUND/i, severity: 'high', category: 'network' },
  { pattern: /Cannot find module|MODULE_NOT_FOUND/i, severity: 'critical', category: 'dependency' },
  { pattern: /Port \d+ is already in use/i, severity: 'critical', category: 'port_conflict' },
  { pattern: /EADDRINUSE/i, severity: 'critical', category: 'port_conflict' },
  { pattern: /Memory usage|heap out of memory/i, severity: 'critical', category: 'memory' },
  { pattern: /401|403|500|502|503|504/i, severity: 'high', category: 'http_error' },
  { pattern: /restart/i, severity: 'medium', category: 'restart' },
  { pattern: /Unhandled|UnhandledPromiseRejection/i, severity: 'high', category: 'unhandled' },
];

// Middleware
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
declare module 'express-session' {
  interface SessionData {
    authenticated: boolean;
  }
}

app.use(
  session({
    secret: 'dw-agents-log-monitor-2025',
    resave: false,
    saveUninitialized: true,
    rolling: true,
    cookie: {
      secure: false,
      httpOnly: true,
      maxAge: 7 * 24 * 60 * 60 * 1000,
    },
  })
);

// Authentication
// REMOVED: app.post('/login', handleLogin);

// REMOVED: const requireAuth = (req: Request, res: Response, next: any) => { ... };


// Store detected issues
interface DetectedIssue {
  id: string;
  timestamp: string;
  agent: string;
  severity: 'critical' | 'high' | 'medium' | 'low';
  category: string;
  message: string;
  logFile: string;
  taskCreated: boolean;
  taskId?: string;
  resolved: boolean;
}

let detectedIssues: DetectedIssue[] = [];

// Load existing issues
function loadIssues() {
  if (fs.existsSync(ISSUES_FILE)) {
    try {
      const data = JSON.parse(fs.readFileSync(ISSUES_FILE, 'utf-8'));
      // Handle both old format (array) and new format (object with issues property)
      if (Array.isArray(data)) {
        detectedIssues = data;
      } else if (data && Array.isArray(data.issues)) {
        detectedIssues = data.issues;
      } else {
        detectedIssues = [];
      }
    } catch (e) {
      console.error('Error loading issues:', e);
      detectedIssues = [];
    }
  }
}

function saveIssues() {
  // Save in simple array format for compatibility
  fs.writeFileSync(ISSUES_FILE, JSON.stringify(detectedIssues, null, 2));
}

loadIssues();

// Analyze log entry
function analyzeLogEntry(logLine: string, agentName: string, logFile: string): DetectedIssue | null {
  for (const pattern of ERROR_PATTERNS) {
    if (pattern.pattern.test(logLine)) {
      return {
        id: `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`,
        timestamp: new Date().toISOString(),
        agent: agentName,
        severity: pattern.severity as any,
        category: pattern.category,
        message: logLine.substring(0, 500),
        logFile,
        taskCreated: false,
        resolved: false,
      };
    }
  }
  return null;
}

// Create task in orchestrator
async function createTask(issue: DetectedIssue) {
  const task = {
    id: `task-${issue.id}`,
    title: `[${issue.severity.toUpperCase()}] ${issue.category} in ${issue.agent}`,
    description: `Automated detection from log monitor:\n\n${issue.message}\n\nLog: ${issue.logFile}`,
    status: 'pending',
    priority: issue.severity === 'critical' ? 'critical' : issue.severity === 'high' ? 'high' : 'medium',
    assignedAgent: determineAgent(issue),
    createdAt: new Date().toISOString(),
    autoGenerated: true,
    source: 'log-monitor',
    issueId: issue.id,
  };

  // Save to tasks database
  let tasks: any[] = [];
  if (fs.existsSync(TASKS_FILE)) {
    try {
      const data = JSON.parse(fs.readFileSync(TASKS_FILE, 'utf-8'));
      tasks = data.tasks || [];
    } catch (e) {
      tasks = [];
    }
  }

  tasks.push(task);
  fs.writeFileSync(TASKS_FILE, JSON.stringify({ tasks }, null, 2));

  issue.taskCreated = true;
  issue.taskId = task.id;

  console.log(`✅ Created task ${task.id} for ${issue.agent} - ${issue.category}`);
  return task.id;
}

// Determine which agent should handle the issue
function determineAgent(issue: DetectedIssue): string {
  if (issue.severity === 'critical') return 'needs-attention';
  if (issue.category === 'port_conflict' || issue.category === 'memory') return 'server-uptime';
  if (issue.agent === 'claude-code') return 'task-orchestrator'; // Claude Code errors go to orchestrator
  if (issue.agent.includes('shopify')) return 'shopify-store';
  if (issue.agent.includes('purchasing')) return 'purchasing-office';
  if (issue.agent.includes('marketing')) return 'marketing';
  if (issue.agent.includes('zendesk')) return 'zendesk-chat';
  if (issue.agent.includes('accounting')) return 'accounting';
  if (issue.agent.includes('legal')) return 'legal-team';

  return 'task-orchestrator'; // Default
}

// Monitor logs continuously
async function monitorLogs() {
  try {
    // Monitor DW-Agents logs
    const logFiles = fs.readdirSync(LOGS_DIR).filter(f => f.endsWith('.log'));

    for (const logFile of logFiles) {
      const filePath = path.join(LOGS_DIR, logFile);
      const agentName = logFile.replace('-error.log', '').replace('-out.log', '');

      // Read last 100 lines
      try {
        const { stdout } = await execAsync(`tail -n 100 "${filePath}"`);
        const lines = stdout.split('\n').filter(l => l.trim());

        for (const line of lines) {
          const issue = analyzeLogEntry(line, agentName, logFile);

          if (issue) {
            // Check if we already detected this recently (avoid duplicates)
            const recentDuplicate = detectedIssues.find(
              i => i.agent === issue.agent &&
                   i.category === issue.category &&
                   i.message === issue.message &&
                   Date.now() - new Date(i.timestamp).getTime() < 5 * 60 * 1000 // 5 min
            );

            if (!recentDuplicate) {
              detectedIssues.push(issue);
              console.log(`🚨 Detected ${issue.severity} issue in ${issue.agent}: ${issue.category}`);

              // Create task for high and critical issues
              if (issue.severity === 'high' || issue.severity === 'critical') {
                await createTask(issue);
              }
            }
          }
        }
      } catch (error) {
        // Skip if file doesn't exist or can't be read
      }
    }

    // Monitor Claude Code debug logs
    try {
      if (fs.existsSync(CLAUDE_DEBUG_DIR)) {
        const claudeDebugFiles = fs.readdirSync(CLAUDE_DEBUG_DIR)
          .filter(f => f.endsWith('.txt'))
          .sort((a, b) => {
            const statA = fs.statSync(path.join(CLAUDE_DEBUG_DIR, a));
            const statB = fs.statSync(path.join(CLAUDE_DEBUG_DIR, b));
            return statB.mtimeMs - statA.mtimeMs;
          })
          .slice(0, 10); // Check only 10 most recent files

        for (const debugFile of claudeDebugFiles) {
          const filePath = path.join(CLAUDE_DEBUG_DIR, debugFile);
          const stats = fs.statSync(filePath);

          // Only check files modified in last hour
          if (Date.now() - stats.mtimeMs < 60 * 60 * 1000) {
            try {
              const { stdout } = await execAsync(`tail -n 200 "${filePath}"`);
              const lines = stdout.split('\n').filter(l => l.trim());

              for (const line of lines) {
                const issue = analyzeLogEntry(line, 'claude-code', `debug/${debugFile}`);

                if (issue) {
                  const recentDuplicate = detectedIssues.find(
                    i => i.agent === 'claude-code' &&
                         i.category === issue.category &&
                         i.message === issue.message &&
                         Date.now() - new Date(i.timestamp).getTime() < 5 * 60 * 1000
                  );

                  if (!recentDuplicate) {
                    detectedIssues.push(issue);
                    console.log(`🚨 Detected ${issue.severity} Claude Code issue: ${issue.category}`);

                    // Create task for high and critical issues
                    if (issue.severity === 'high' || issue.severity === 'critical') {
                      await createTask(issue);
                    }
                  }
                }
              }
            } catch (error) {
              // Skip if file can't be read
            }
          }
        }
      }
    } catch (error) {
      // Claude debug dir might not exist
    }

    // Keep only last 1000 issues
    if (detectedIssues.length > 1000) {
      detectedIssues = detectedIssues.slice(-1000);
    }

    saveIssues();
  } catch (error) {
    console.error('Error monitoring logs:', error);
  }
}

// Check system CPU usage
async function checkSystemResources() {
  try {
    // Get CPU usage
    const { stdout: cpuInfo } = await execAsync("top -bn1 | grep 'Cpu(s)' | awk '{print $2}' | cut -d'%' -f1");
    const cpuUsage = parseFloat(cpuInfo.trim());
    
    // Get memory usage
    const { stdout: memInfo } = await execAsync("free -m | grep Mem | awk '{print $3/$2 * 100.0}'");
    const memUsage = parseFloat(memInfo.trim());
    
    // Check if CPU usage is too high (>90%)
    if (cpuUsage > 90) {
      const issue: DetectedIssue = {
        id: `cpu-critical-${Date.now()}`,
        timestamp: new Date().toISOString(),
        agent: 'system',
        severity: 'critical',
        category: 'high_cpu',
        message: `CRITICAL: CPU usage at ${cpuUsage.toFixed(1)}% - Exceeds 90% threshold! Consider upgrading Kamatera server CPU.`,
        logFile: 'system',
        taskCreated: false,
        resolved: false,
      };
      
      // Check if we already reported this in last 5 minutes
      const recentCpuAlert = detectedIssues.find(
        i => i.category === 'high_cpu' && 
             Date.now() - new Date(i.timestamp).getTime() < 5 * 60 * 1000
      );
      
      if (!recentCpuAlert) {
        detectedIssues.push(issue);
        await createTask(issue);
        console.error(`🚨 CRITICAL CPU ALERT: ${cpuUsage.toFixed(1)}% usage!`);
      }
    } else if (cpuUsage > 75) {
      const issue: DetectedIssue = {
        id: `cpu-high-${Date.now()}`,
        timestamp: new Date().toISOString(),
        agent: 'system',
        severity: 'high',
        category: 'high_cpu',
        message: `WARNING: CPU usage at ${cpuUsage.toFixed(1)}% - Approaching critical threshold`,
        logFile: 'system',
        taskCreated: false,
        resolved: false,
      };
      
      const recentCpuWarning = detectedIssues.find(
        i => i.category === 'high_cpu' && i.severity === 'high' &&
             Date.now() - new Date(i.timestamp).getTime() < 10 * 60 * 1000
      );
      
      if (!recentCpuWarning) {
        detectedIssues.push(issue);
      }
    }
    
    // Store current metrics for dashboard
    (global as any).systemMetrics = {
      cpuUsage: cpuUsage.toFixed(1),
      memUsage: memUsage.toFixed(1),
      timestamp: new Date().toISOString()
    };
    
  } catch (error) {
    console.error('Error checking system resources:', error);
  }
}

// Check PM2 process status
async function checkPM2Status() {
  try {
    const { stdout } = await execAsync('pm2 jlist');
    const processes = JSON.parse(stdout);

    for (const proc of processes) {
      // Check for frequent restarts
      if (proc.pm2_env.restart_time > 5 && proc.pm2_env.uptime < 60000) {
        const issue: DetectedIssue = {
          id: `pm2-${Date.now()}`,
          timestamp: new Date().toISOString(),
          agent: proc.name,
          severity: 'critical',
          category: 'frequent_restarts',
          message: `Process ${proc.name} has restarted ${proc.pm2_env.restart_time} times recently`,
          logFile: 'pm2',
          taskCreated: false,
          resolved: false,
        };

        detectedIssues.push(issue);
        await createTask(issue);
      }

      // Check memory usage
      if (proc.monit && proc.monit.memory > 400 * 1024 * 1024) { // > 400MB
        const issue: DetectedIssue = {
          id: `mem-${Date.now()}`,
          timestamp: new Date().toISOString(),
          agent: proc.name,
          severity: 'high',
          category: 'high_memory',
          message: `Process ${proc.name} using ${Math.round(proc.monit.memory / 1024 / 1024)}MB memory`,
          logFile: 'pm2',
          taskCreated: false,
          resolved: false,
        };

        const recentDuplicate = detectedIssues.find(
          i => i.agent === issue.agent && i.category === 'high_memory' &&
               Date.now() - new Date(i.timestamp).getTime() < 10 * 60 * 1000
        );

        if (!recentDuplicate) {
          detectedIssues.push(issue);
          await createTask(issue);
        }
      }
    }

    saveIssues();
  } catch (error) {
    console.error('Error checking PM2 status:', error);
  }
}

// Start monitoring interval
setInterval(monitorLogs, 30000); // Every 30 seconds
setInterval(checkPM2Status, 60000); // Every 1 minute
setInterval(checkSystemResources, 10000); // Every 10 seconds for CPU monitoring

// Initial scan
monitorLogs();
checkPM2Status();
checkSystemResources();

// Dashboard
app.get('/', requireGlobalAuth, (req: Request, res: Response) => {
  const criticalCount = detectedIssues.filter(i => !i.resolved && i.severity === 'critical').length;
  const highCount = detectedIssues.filter(i => !i.resolved && i.severity === 'high').length;
  const mediumCount = detectedIssues.filter(i => !i.resolved && i.severity === 'medium').length;
  
  // Get system metrics
  const metrics = (global as any).systemMetrics || { cpuUsage: '0', memUsage: '0' };
  const cpuUsage = parseFloat(metrics.cpuUsage);
  const cpuClass = cpuUsage > 90 ? 'critical' : cpuUsage > 75 ? 'high' : 'normal';

  res.send(`
    <!DOCTYPE html>
    <html>
    <head>
      <title>Log Monitor Agent - Dashboard</title>
      ${getThemeStyles('modern-minimalist')}
      <style>
        * { margin: 0; padding: 0; box-sizing: border-box; }
        body {
          font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
          background: #f5f5f5;
        }
        .header {
          background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
          color: white;
          padding: 20px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
        }
        .header h1 { font-size: 24px; margin-bottom: 10px; }
        .header p { font-size: 14px; opacity: 0.9; }
        .stats {
          display: flex;
          gap: 20px;
          padding: 20px;
          flex-wrap: wrap;
        }
        .stat-box {
          background: white;
          padding: 20px;
          border-radius: 10px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
          min-width: 150px;
          text-align: center;
        }
        .stat-box h3 {
          font-size: 14px;
          color: #666;
          margin-bottom: 10px;
        }
        .stat-box p {
          font-size: 32px;
          font-weight: bold;
        }
        .stat-box.critical p { color: #e74c3c; }
        .stat-box.high p { color: #e67e22; }
        .stat-box.medium p { color: #f39c12; }
        .stat-box.total p { color: #3498db; }
        .stat-box.cpu p { color: #9b59b6; }
        .stat-box.cpu.critical { background: #ffe0e0; }
        .stat-box.cpu.critical p { color: #e74c3c; font-size: 48px; }
        .stat-box.cpu.high { background: #fff4e0; }
        .stat-box.cpu.high p { color: #e67e22; }
        .stat-box.normal p { color: #27ae60; }
        .stat-box.memory p { color: #2980b9; }
        .content {
          margin: 20px;
          background: white;
          padding: 20px;
          border-radius: 10px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
        }
        table {
          width: 100%;
          border-collapse: collapse;
          margin-top: 15px;
        }
        th, td {
          padding: 12px;
          text-align: left;
          border-bottom: 1px solid #eee;
          font-size: 13px;
        }
        th {
          background: #f9f9f9;
          font-weight: 600;
        }
        tr:hover { background: #fafafa; }
        .severity {
          padding: 4px 12px;
          border-radius: 12px;
          font-size: 11px;
          font-weight: bold;
          text-transform: uppercase;
        }
        .severity.critical {
          background: #fee;
          color: #e74c3c;
        }
        .severity.high {
          background: #fef3e0;
          color: #e67e22;
        }
        .severity.medium {
          background: #fef9e0;
          color: #f39c12;
        }
        .severity.low {
          background: #e8f5e9;
          color: #27ae60;
        }
        .badge {
          display: inline-block;
          padding: 4px 8px;
          border-radius: 4px;
          font-size: 11px;
          background: #3498db;
          color: white;
        }
        .badge.task { background: #27ae60; }
        .badge.resolved { background: #95a5a6; }
        .btn {
          padding: 6px 12px;
          background: #3498db;
          color: white;
          border: none;
          border-radius: 4px;
          cursor: pointer;
          font-size: 12px;
        }
        .btn:hover { background: #2980b9; }
        .btn.resolve { background: #27ae60; }
        .btn.resolve:hover { background: #229954; }
        @keyframes spin {
          0% { transform: rotate(0deg); }
          100% { transform: rotate(360deg); }
        }
        #scroll-to-top:hover {
          background: #2980b9;
          transform: translateY(-2px);
          box-shadow: 0 6px 16px rgba(0,0,0,0.3);
        }
        .hooks-panel {
          background: white;
          padding: 20px;
          margin: 20px;
          border-radius: 10px;
          box-shadow: 0 2px 10px rgba(0,0,0,0.1);
        }
        .hooks-panel h3 {
          margin: 0 0 15px 0;
          font-size: 16px;
          color: #666;
        }
        .hook-btn {
          padding: 10px 20px;
          margin-right: 10px;
          background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
          color: white;
          border: none;
          border-radius: 6px;
          cursor: pointer;
          font-size: 13px;
          font-weight: 600;
          transition: all 0.2s;
        }
        .hook-btn:hover {
          transform: translateY(-2px);
          box-shadow: 0 4px 8px rgba(240, 147, 251, 0.4);
        }
        .hook-btn:disabled {
          opacity: 0.5;
          cursor: not-allowed;
        }
        .hook-btn.fix {
          background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
        }
        .hook-btn.monitor {
          background: linear-gradient(135deg, #4facfe 0%, #00f2fe 100%);
        }
      </style>
    </head>
    <body style="padding: 20px;">
      ${getUniversalHeader('Log Monitor', PORT, 5)}

      <div class="header" style="margin-top: 20px;">
        <h1>📋 Log Monitor Agent</h1>
        <p>Real-time log analysis and automated issue detection</p>
        <p>Monitoring: DW-Agents logs (/root/Projects/Designer-Wallcoverings/DW-Agents/logs) + Claude Code debug logs (/root/.claude/debug)</p>
      </div>

      <div class="hooks-panel">
        <h3>🔧 System Tools</h3>
        <button class="hook-btn" onclick="runHook('service-health-check', 'check')">🏥 Health Check</button>
        <button class="hook-btn fix" onclick="runHook('service-health-check', 'fix')">🔧 Auto-Fix</button>
        <button class="hook-btn monitor" onclick="runHook('agent-health-monitor')">👥 Monitor Agents</button>
      </div>

      <div class="stats">
        <div class="stat-box critical">
          <h3>Critical Issues</h3>
          <p>${criticalCount}</p>
        </div>
        <div class="stat-box high">
          <h3>High Priority</h3>
          <p>${highCount}</p>
        </div>
        <div class="stat-box medium">
          <h3>Medium Priority</h3>
          <p>${mediumCount}</p>
        </div>
        <div class="stat-box total">
          <h3>Total Detected</h3>
          <p>${detectedIssues.length}</p>
        </div>
      </div>

      <div class="content">
        <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 15px;">
          <h2>Recent Issues <span id="issue-count" style="color: #666; font-size: 14px;">(Loading...)</span></h2>
          <button onclick="manualRefresh()" class="btn" style="padding: 10px 20px; font-size: 14px; display: flex; align-items: center; gap: 8px;">
            <span id="refresh-icon">🔄</span> Refresh Data
          </button>
        </div>
        <div id="table-container" style="max-height: 600px; overflow-y: auto; position: relative;">
          <table id="issues-table">
            <thead style="position: sticky; top: 0; background: white; z-index: 10;">
              <tr>
                <th>Time</th>
                <th>Agent</th>
                <th>Severity</th>
                <th>Category</th>
                <th>Message</th>
                <th>Status</th>
                <th>Actions</th>
              </tr>
            </thead>
            <tbody id="issues-tbody">
              <!-- Issues will be loaded dynamically -->
            </tbody>
          </table>
          <div id="loading-indicator" style="display: none; text-align: center; padding: 20px; color: #666;">
            <div style="display: inline-block; width: 20px; height: 20px; border: 3px solid #f3f3f3; border-top: 3px solid #3498db; border-radius: 50%; animation: spin 1s linear infinite;"></div>
            <p style="margin-top: 10px;">Loading issues...</p>
          </div>

          <!-- Pagination Controls -->
          <div id="pagination-controls" style="display: flex; justify-content: center; align-items: center; padding: 30px; gap: 20px;">
            <button id="prev-btn" onclick="loadPreviousPage()"
              style="padding: 12px 30px; background: #3498db; color: white; border: none; border-radius: 8px; cursor: pointer; font-size: 16px; font-weight: bold; transition: all 0.3s ease; box-shadow: 0 2px 8px rgba(52,152,219,0.3);">
              ← Back
            </button>
            <div id="page-info" style="font-size: 16px; color: #666; font-weight: 500;">
              Page 1
            </div>
            <button id="next-btn" onclick="loadNextPage()"
              style="padding: 12px 30px; background: #3498db; color: white; border: none; border-radius: 8px; cursor: pointer; font-size: 16px; font-weight: bold; transition: all 0.3s ease; box-shadow: 0 2px 8px rgba(52,152,219,0.3);">
              Next →
            </button>
          </div>
        </div>
      </div>

      <button id="scroll-to-top" style="display: none; position: fixed; bottom: 30px; right: 30px; background: #3498db; color: white; border: none; padding: 15px 20px; border-radius: 50px; cursor: pointer; box-shadow: 0 4px 12px rgba(0,0,0,0.2); font-size: 14px; font-weight: bold; z-index: 1000; transition: all 0.3s ease;">
        ↑ Back to Top
      </button>

      <script>
        // Pagination state management
        let currentOffset = 0;
        let isLoading = false;
        let hasMore = true;
        let totalIssues = 0;
        const LIMIT = 250;
        let loadedIssueIds = new Set(); // Track loaded issues to prevent duplicates

        // Create issue row HTML
        function createIssueRow(issue) {
          return \`
            <tr data-issue-id="\${issue.id}">
              <td>\${new Date(issue.timestamp).toLocaleString()}</td>
              <td><strong>\${issue.agent}</strong></td>
              <td><span class="severity \${issue.severity}">\${issue.severity}</span></td>
              <td>\${issue.category}</td>
              <td style="max-width: 400px; overflow: hidden; text-overflow: ellipsis; white-space: nowrap;">\${issue.message}</td>
              <td>
                \${issue.resolved ? '<span class="badge resolved">Resolved</span>' : ''}
                \${issue.taskCreated ? \`<span class="badge task">Task \${issue.taskId}</span>\` : ''}
              </td>
              <td>
                \${!issue.resolved ? \`<button class="btn resolve" onclick="resolveIssue('\${issue.id}')">Mark Resolved</button>\` : ''}
                \${!issue.taskCreated ? \`<button class="btn" onclick="createTask('\${issue.id}')">Create Task</button>\` : ''}
              </td>
            </tr>
          \`;
        }

        // Load issues from API
        async function loadIssues() {
          if (isLoading) return;

          isLoading = true;
          const loadingIndicator = document.getElementById('loading-indicator');
          loadingIndicator.style.display = 'block';

          try {
            const response = await fetch(\`/api/issues?limit=\${LIMIT}&offset=\${currentOffset}\`);
            const data = await response.json();

            const tbody = document.getElementById('issues-tbody');
            tbody.innerHTML = '';
            loadedIssueIds.clear();

            // Add rows
            data.issues.forEach(issue => {
              if (!loadedIssueIds.has(issue.id)) {
                loadedIssueIds.add(issue.id);
                tbody.insertAdjacentHTML('beforeend', createIssueRow(issue));
              }
            });

            // Update state
            hasMore = data.hasMore;
            totalIssues = data.total;

            // Update pagination controls
            updatePaginationControls();

            // Update counter
            const start = currentOffset + 1;
            const end = Math.min(currentOffset + data.issues.length, totalIssues);
            document.getElementById('issue-count').textContent =
              \`(Showing \${start}-\${end} of \${totalIssues})\`;

            loadingIndicator.style.display = 'none';

            // Don't auto-scroll - let user maintain their position

          } catch (error) {
            console.error('Error loading issues:', error);
            loadingIndicator.innerHTML = '<p style="color: #e74c3c;">Error loading issues</p>';
          } finally {
            isLoading = false;
          }
        }

        // Update pagination button states
        function updatePaginationControls() {
          const prevBtn = document.getElementById('prev-btn');
          const nextBtn = document.getElementById('next-btn');
          const pageInfo = document.getElementById('page-info');

          const currentPage = Math.floor(currentOffset / LIMIT) + 1;
          const totalPages = Math.ceil(totalIssues / LIMIT);

          pageInfo.textContent = \`Page \${currentPage} of \${totalPages}\`;

          // Disable/enable buttons
          prevBtn.disabled = currentOffset === 0;
          nextBtn.disabled = !hasMore;

          // Style disabled buttons
          prevBtn.style.opacity = prevBtn.disabled ? '0.5' : '1';
          prevBtn.style.cursor = prevBtn.disabled ? 'not-allowed' : 'pointer';
          nextBtn.style.opacity = nextBtn.disabled ? '0.5' : '1';
          nextBtn.style.cursor = nextBtn.disabled ? 'not-allowed' : 'pointer';
        }

        // Load next page
        function loadNextPage() {
          if (!hasMore || isLoading) return;
          currentOffset += LIMIT;
          loadIssues();
        }

        // Load previous page
        function loadPreviousPage() {
          if (currentOffset === 0 || isLoading) return;
          currentOffset = Math.max(0, currentOffset - LIMIT);
          loadIssues();
        }

        // Removed infinite scroll - now using pagination buttons

        // Scroll to top button functionality
        const scrollToTopBtn = document.getElementById('scroll-to-top');
        const tableContainer = document.getElementById('table-container');

        tableContainer.addEventListener('scroll', () => {
          if (tableContainer.scrollTop > 300) {
            scrollToTopBtn.style.display = 'block';
          } else {
            scrollToTopBtn.style.display = 'none';
          }
        });

        scrollToTopBtn.addEventListener('click', () => {
          tableContainer.scrollTo({
            top: 0,
            behavior: 'smooth'
          });
        });

        // Handle resolve and create task actions
        function resolveIssue(id) {
          fetch('/api/resolve/' + id, { method: 'POST' })
            .then(() => {
              // Update UI without full reload
              const row = document.querySelector(\`tr[data-issue-id="\${id}"]\`);
              if (row) {
                const statusCell = row.cells[5];
                statusCell.innerHTML = '<span class="badge resolved">Resolved</span>';
                const actionsCell = row.cells[6];

                // Remove "Mark Resolved" button
                actionsCell.innerHTML = actionsCell.innerHTML.replace(/<button[^>]*Mark Resolved<\\/button>/, '');

                // Change "Create Task" button from blue to green
                const createTaskBtn = actionsCell.querySelector('button');
                if (createTaskBtn && createTaskBtn.textContent.includes('Create Task')) {
                  createTaskBtn.classList.add('resolve'); // Add green class
                  createTaskBtn.style.background = '#27ae60'; // Green color
                }
              }
            });
        }

        function createTask(id) {
          fetch('/api/create-task/' + id, { method: 'POST' })
            .then(res => res.json())
            .then(data => {
              if (data.success) {
                // Update UI without full reload
                const row = document.querySelector(\`tr[data-issue-id="\${id}"]\`);
                if (row) {
                  const statusCell = row.cells[5];
                  statusCell.innerHTML += \`<span class="badge task">Task \${data.taskId}</span>\`;
                  const actionsCell = row.cells[6];
                  actionsCell.innerHTML = actionsCell.innerHTML.replace(/<button[^>]*Create Task<\\/button>/, '');
                }
              }
            });
        }

        // Lightweight check for new issues (every 60 seconds - less aggressive)
        let lastTotalIssues = 0;
        setInterval(async () => {
          try {
            const response = await fetch('/api/issues?limit=1&offset=0');
            const data = await response.json();

            // If new issues detected, show a non-intrusive notification badge
            if (data.total > lastTotalIssues && lastTotalIssues > 0) {
              const newIssuesCount = data.total - lastTotalIssues;

              // Show subtle notification instead of auto-reloading
              showNewIssuesNotification(newIssuesCount);

              console.log(\`New issues detected: \${newIssuesCount}\`);
            }
            lastTotalIssues = data.total;
          } catch (error) {
            console.error('Error checking for new issues:', error);
          }
        }, 60000); // Reduced from 30s to 60s

        // Show non-intrusive notification for new issues
        function showNewIssuesNotification(count) {
          // Remove existing notification if present
          const existing = document.getElementById('new-issues-notification');
          if (existing) existing.remove();

          const notification = document.createElement('div');
          notification.id = 'new-issues-notification';
          notification.style.cssText = \`
            position: fixed;
            top: 100px;
            right: 30px;
            background: linear-gradient(135deg, #f093fb 0%, #f5576c 100%);
            color: white;
            padding: 15px 25px;
            border-radius: 10px;
            box-shadow: 0 4px 12px rgba(0,0,0,0.3);
            z-index: 10000;
            cursor: pointer;
            font-weight: bold;
            animation: slideIn 0.3s ease;
            transition: transform 0.2s;
          \`;
          notification.innerHTML = \`
            <div style="display: flex; align-items: center; gap: 10px;">
              <span style="font-size: 20px;">🔔</span>
              <div>
                <div style="font-size: 14px;">\${count} New Issue\${count > 1 ? 's' : ''}</div>
                <div style="font-size: 11px; opacity: 0.9;">Click to refresh</div>
              </div>
            </div>
          \`;

          notification.addEventListener('click', () => {
            currentOffset = 0;
            loadIssues();
            notification.remove();
          });

          notification.addEventListener('mouseenter', () => {
            notification.style.transform = 'translateY(-2px)';
          });

          notification.addEventListener('mouseleave', () => {
            notification.style.transform = 'translateY(0)';
          });

          document.body.appendChild(notification);

          // Add animation keyframes if not exists
          if (!document.getElementById('notification-animations')) {
            const style = document.createElement('style');
            style.id = 'notification-animations';
            style.textContent = \`
              @keyframes slideIn {
                from {
                  transform: translateX(400px);
                  opacity: 0;
                }
                to {
                  transform: translateX(0);
                  opacity: 1;
                }
              }
            \`;
            document.head.appendChild(style);
          }
        }

        // Manual refresh function with visual feedback
        async function manualRefresh() {
          const icon = document.getElementById('refresh-icon');
          const originalIcon = icon.textContent;

          // Rotate animation
          icon.style.display = 'inline-block';
          icon.style.animation = 'spin 1s linear infinite';
          icon.textContent = '⏳';

          // Clear notification if present
          const notification = document.getElementById('new-issues-notification');
          if (notification) notification.remove();

          // Reset to first page and reload
          currentOffset = 0;
          await loadIssues();

          // Reset icon
          setTimeout(() => {
            icon.style.animation = '';
            icon.textContent = '✓';
            setTimeout(() => {
              icon.textContent = originalIcon;
            }, 1000);
          }, 500);
        }

        // Initial load
        loadIssues(false).then(() => {
          lastTotalIssues = totalIssues;
        });

        async function runHook(hookName, action = '') {
          const btn = event.target;
          if (btn.disabled) return;

          btn.disabled = true;
          const originalText = btn.innerHTML;
          btn.innerHTML = '⏳ Running...';

          try {
            const response = await fetch('/api/hooks/' + hookName, {
              method: 'POST',
              headers: { 'Content-Type': 'application/json' },
              body: JSON.stringify({ action })
            });

            const result = await response.json();

            if (result.success) {
              alert('✅ Hook executed successfully!\\n\\n' + (result.output || 'No output'));
            } else {
              alert('❌ Hook error: ' + (result.error || 'Unknown error'));
            }
          } catch (err) {
            console.error('Hook execution error:', err);
            alert('❌ Error: ' + err.message);
          } finally {
            btn.disabled = false;
            btn.innerHTML = originalText;
          }
        }
      </script>
    </body>
    </html>
  `);
});

// API: Get issues (paginated for infinite scroll)
app.get('/api/issues', requireGlobalAuth, (req: Request, res: Response) => {
  const limit = parseInt(req.query.limit as string) || 250;
  const offset = parseInt(req.query.offset as string) || 0;

  // Return issues in reverse chronological order (newest first)
  const allIssues = [...detectedIssues].reverse();
  const paginatedIssues = allIssues.slice(offset, offset + limit);

  res.json({
    issues: paginatedIssues,
    total: detectedIssues.length,
    hasMore: offset + limit < detectedIssues.length,
    offset,
    limit
  });
});

// API: Resolve issue
app.post('/api/resolve/:id', requireGlobalAuth, (req: Request, res: Response) => {
  const issue = detectedIssues.find(i => i.id === req.params.id);
  if (issue) {
    issue.resolved = true;
    saveIssues();
  }
  res.json({ success: true });
});

// API: Create task manually
app.post('/api/create-task/:id', requireGlobalAuth, async (req: Request, res: Response) => {
  const issue = detectedIssues.find(i => i.id === req.params.id);
  if (issue && !issue.taskCreated) {
    await createTask(issue);
    saveIssues();
    res.json({ success: true, taskId: issue.taskId });
  } else {
    res.json({ success: false });
  }
});

// API: Run system hooks
app.post('/api/hooks/:hookName', requireGlobalAuth, async (req: Request, res: Response) => {
  const { hookName } = req.params;
  const { action } = req.body;

  try {
    const cmd = `node /root/Projects/Designer-Wallcoverings/DW-MCP/DW-Hooks/${hookName}-hook.js ${action || ''}`.trim();

    exec(cmd, { timeout: 30000 }, (error, stdout, stderr) => {
      if (error) {
        console.error('Hook execution error:', error);
        return res.status(500).json({
          success: false,
          error: stderr || error.message
        });
      }

      res.json({
        success: true,
        output: stdout
      });
    });
  } catch (err: any) {
    console.error('Hook error:', err);
    res.status(500).json({
      success: false,
      error: err.message
    });
  }
});

// Health check
app.get('/health', (req: Request, res: Response) => {
  res.json({
    status: 'healthy',
    agent: 'log-monitor',
    port: PORT,
    issuesDetected: detectedIssues.length,
    unresolvedCritical: detectedIssues.filter(i => !i.resolved && i.severity === 'critical').length,
    timestamp: new Date().toISOString()
  });
});

// Start server

// Metrics endpoint for daily reporting
app.get("/api/metrics", (req: Request, res: Response) => {
  res.json({
    status: "online",
    uptime: process.uptime(),
    responseTime: 0,
    tasksCompleted: 0,
    errorsToday: 0,
    lastActivity: new Date().toISOString(),
    qnaReadiness: 100
  });
});

// Global error handler - MUST be last middleware
app.use((err: any, req: Request, res: Response, next: any) => {
  console.error('🚨 EXPRESS ERROR HANDLER CAUGHT:', err);
  console.error('Stack:', err.stack);
  console.error('Request URL:', req.url);
  console.error('Request method:', req.method);

  res.status(500).json({
    error: 'Internal Server Error',
    message: err.message,
    stack: process.env.NODE_ENV === 'development' ? err.stack : undefined
  });
});

app.listen(PORT, () => {
  console.log(`📋 Log Monitor Agent running on port ${PORT}`);
  console.log(`📊 Dashboard: http://localhost:${PORT}`);
  console.log(`🔍 Monitoring ${LOGS_DIR}`);
});

// Graceful shutdown
process.on('SIGTERM', () => {
  console.log('SIGTERM received, shutting down gracefully...');
  saveIssues();
  process.exit(0);
});

process.on('SIGINT', () => {
  console.log('SIGINT received, shutting down gracefully...');
  saveIssues();
  process.exit(0);
});