← back to Designer Wallcoverings

DW-Agents/dw-agents/agent-server-uptime/server-uptime-agent.ts.backup-oauth-20251112

878 lines

import cookieParser from "cookie-parser";
/**
 * DW-Agents: Server Uptime Agent
 *
 * Monitors all processes and ensures they never crash
 * - Checks all PM2 processes every 15 seconds
 * - Auto-restarts any stopped/errored processes immediately
 * - Monitors process health (CPU, memory, restart counts)
 * - Alerts on critical issues
 * - Ensures 100% uptime across all agents
 *
 * Port: 9888
 */

import express, { Request, Response } from 'express';
import { requireAuth as ssoAuth, handleLogin, setSSOToken, SSO_TOKEN_NAME, SSO_TOKEN_VALUE } from './shared-auth';
import cookieParser from 'cookie-parser';
import session from 'express-session';
import { exec } from 'child_process';
import { promisify } from 'util';
import fs from 'fs';
import path from 'path';

import { chatMiddleware } from '../shared-chat-integration';

const execAsync = promisify(exec);

// Load known fixes
const KNOWN_FIXES_FILE = path.join(__dirname, 'known-fixes.json');
let knownFixes: any = { fixes: [] };
try {
  if (fs.existsSync(KNOWN_FIXES_FILE)) {
    knownFixes = JSON.parse(fs.readFileSync(KNOWN_FIXES_FILE, 'utf8'));
  }
} catch (error) {
  console.error('Failed to load known-fixes.json:', error);
}

const app = express();

// Global Authentication
const PORT = 9888;

// Global Authentication - MUST come before any other middleware
app.use(cookieParser());

// Add inter-agent chat widget (after auth)
app.use(chatMiddleware({
  agentId: 'agent-server-uptime',
  agentName: 'Server Uptime',
  port: 9888,
  category: 'agent'
}));


app.use(express.json());
app.use(express.urlencoded({ extended: true }));

declare module 'express-session' {
  interface SessionData {
    authenticated: boolean;
  }
}

app.use(
  session({
    secret: 'dw-server-uptime-2025',
    resave: false,
    saveUninitialized: true,
    cookie: {
      secure: false,
      httpOnly: true,
      maxAge: 7 * 24 * 60 * 60 * 1000,
    },
  })
);

const requireAuth = (req: Request, res: Response, next: any) => {
  // Check session first
  if (req.session.authenticated) {
    return next();
  }

  // Check SSO cookie
  if (req.cookies && req.cookies[SSO_TOKEN_NAME] === SSO_TOKEN_VALUE) {
    req.session.authenticated = true;
    return next();
  }

  res.redirect('/login');
};

// Live domains to monitor with their expected backend ports
const LIVE_DOMAINS = [
  'cypresawards.com',
  'flockedwallpaper.com',
  'glassbeadedwallpaper.com',
  'goodquestion.ai',
  'grassclothwallcoverings.com',
  'grassclothwallpaper.com',
  'latinogardens.greendomainbrokers.com',
  'losgardeners.com',
  'n8n.greendomainbrokers.com',
  'novasuede.com',
  'www.cypresawards.com',
  'www.flockedwallpaper.com',
  'www.glassbeadedwallpaper.com',
  'www.goodquestion.ai',
  'www.grassclothwallcoverings.com',
  'www.grassclothwallpaper.com',
  'www.losgardeners.com',
  'www.novasuede.com'
];

// Expected nginx port mappings (domain -> correct backend port)
const NGINX_PORT_MAPPINGS: { [key: string]: number } = {
  'grassclothwallpaper.com': 3201,
  'www.grassclothwallpaper.com': 3201,
  'flockedwallpaper.com': 3200,
  'www.flockedwallpaper.com': 3200,
  'novasuede.com': 3204,
  'www.novasuede.com': 3204,
  'glassbeadedwallpaper.com': 3203,
  'www.glassbeadedwallpaper.com': 3203
};

// Stats tracking
const stats = {
  totalRestarts: 0,
  lastCheck: new Date(),
  lastDomainCheck: new Date(),
  restartsToday: [] as Array<{ name: string; time: Date; reason: string }>,
  uptime: Date.now(),
  checksPerformed: 0,
  domainChecksPerformed: 0
};

// Domain health tracking
const domainHealth: Map<string, { status: 'up' | 'down' | 'checking'; lastCheck: Date; responseTime?: number }> = new Map();

// Activity log
const activityLog: Array<{ timestamp: Date; action: string; process: string; details: string }> = [];

function logActivity(action: string, process: string, details: string) {
  activityLog.unshift({
    timestamp: new Date(),
    action,
    process,
    details
  });
  if (activityLog.length > 100) activityLog.pop();
}

// Login page
app.get('/login', (req: Request, res: Response) => {
  res.send(`
<!DOCTYPE html>
<html>
<head>
  <title>Server Uptime - Login</title>
  <style>
    * { margin: 0; padding: 0; box-sizing: border-box; }
    body {
      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
      background: linear-gradient(135deg, #00C851 0%, #007E33 100%);
      display: flex;
      align-items: center;
      justify-content: center;
      min-height: 100vh;
    }
    .login-container {
      background: white;
      padding: 40px;
      border-radius: 15px;
      box-shadow: 0 15px 35px rgba(0,0,0,0.2);
      max-width: 400px;
      width: 100%;
    }
    h1 { color: #00C851; text-align: center; margin-bottom: 30px; }
    input {
      width: 100%;
      padding: 12px;
      margin: 10px 0;
      border: 2px solid #e0e0e0;
      border-radius: 8px;
    }
    button {
      width: 100%;
      background: linear-gradient(135deg, #00C851 0%, #007E33 100%);
      color: white;
      border: none;
      padding: 15px;
      border-radius: 8px;
      cursor: pointer;
      font-weight: 600;
      margin-top: 10px;
    }
  </style>
</head>
<body>
  <div class="login-container">
    <h1>🟢 Server Uptime</h1>
    <form method="POST">
      <input type="text" name="username" placeholder="Username" required>
      <input type="password" name="password" placeholder="Password" required>
      <button type="submit">Login</button>
    </form>
  </div>
</body>
</html>
  `);
});

app.post('/login', (req: Request, res: Response) => {
  const { username, password } = req.body;
  if (username === 'admin2025' && password === 'Otis') {
    req.session.authenticated = true;

    // Set SSO cookie for cross-agent authentication
    setSSOToken(res);

    res.redirect('/');
  } else {
    res.redirect('/login');
  }
});

// Main dashboard
app.get('/', requireAuth, (req: Request, res: Response) => {
  res.send(`
<!DOCTYPE html>
<html>
<head>
  <title>Server Uptime Agent - DW-Agents</title>
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <style>
    * { margin: 0; padding: 0; box-sizing: border-box; }
    body {
      font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
      background: linear-gradient(135deg, #00C851 0%, #007E33 100%);
      padding: 20px;
      min-height: 100vh;
    }
    .header {
      background: white;
      padding: 30px;
      border-radius: 15px;
      margin-bottom: 20px;
      box-shadow: 0 5px 20px rgba(0,0,0,0.1);
    }
    .header h1 { color: #00C851; font-size: 2.5em; margin-bottom: 10px; }
    .header .subtitle { color: #666; font-size: 1.1em; }
    .status-bar {
      background: white;
      padding: 15px 30px;
      border-radius: 15px;
      margin-bottom: 20px;
      display: flex;
      justify-content: space-between;
      align-items: center;
      box-shadow: 0 5px 20px rgba(0,0,0,0.1);
    }
    .status-indicator {
      display: flex;
      align-items: center;
      gap: 10px;
      font-weight: 600;
    }
    .status-indicator.healthy { color: #00C851; }
    .pulse {
      width: 12px;
      height: 12px;
      background: #00C851;
      border-radius: 50%;
      animation: pulse 2s infinite;
    }
    @keyframes pulse {
      0%, 100% { opacity: 1; transform: scale(1); }
      50% { opacity: 0.5; transform: scale(1.2); }
    }
    .summary-cards {
      display: grid;
      grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
      gap: 15px;
      margin-bottom: 20px;
    }
    .summary-card {
      background: white;
      padding: 20px;
      border-radius: 12px;
      box-shadow: 0 5px 20px rgba(0,0,0,0.1);
      text-align: center;
    }
    .summary-card .number {
      font-size: 2.5em;
      font-weight: bold;
      color: #00C851;
    }
    .summary-card .label {
      color: #666;
      margin-top: 5px;
    }
    .section {
      background: white;
      padding: 25px;
      border-radius: 15px;
      margin-bottom: 20px;
      box-shadow: 0 5px 20px rgba(0,0,0,0.1);
    }
    .section h2 {
      color: #00C851;
      margin-bottom: 15px;
      font-size: 1.5em;
    }
    .process-table {
      width: 100%;
      border-collapse: collapse;
      font-size: 0.9em;
    }
    .process-table th {
      background: #f8f9fa;
      padding: 12px;
      text-align: left;
      font-weight: 600;
      border-bottom: 2px solid #e0e0e0;
    }
    .process-table td {
      padding: 12px;
      border-bottom: 1px solid #e0e0e0;
    }
    .status-badge {
      display: inline-block;
      padding: 4px 12px;
      border-radius: 12px;
      font-size: 0.85em;
      font-weight: 600;
    }
    .status-online { background: #d1fae5; color: #065f46; }
    .status-stopped { background: #fee2e2; color: #991b1b; }
    .status-errored { background: #fef3c7; color: #92400e; }
    .status-up { background: #d1fae5; color: #065f46; }
    .status-down { background: #fee2e2; color: #991b1b; }
    .domain-grid {
      display: grid;
      grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
      gap: 15px;
      margin-top: 15px;
    }
    .domain-card {
      background: #f8f9fa;
      padding: 15px;
      border-radius: 8px;
      border-left: 4px solid #00C851;
    }
    .domain-card.down {
      border-left-color: #dc143c;
      background: #fff5f5;
    }
    .activity-item {
      background: #f8f9fa;
      padding: 12px;
      border-radius: 8px;
      margin-bottom: 8px;
      border-left: 4px solid #00C851;
    }
    .activity-item .time { color: #888; font-size: 0.85em; }
    .activity-item .action { font-weight: 600; color: #333; }
  </style>
</head>
<body>
  <div class="header">
    <h1>🟢 Server Uptime Agent</h1>
    <div class="subtitle">Automatic process monitoring and recovery - Checks every 15 seconds</div>
  </div>

  <div class="status-bar">
    <div class="status-indicator healthy">
      <div class="pulse"></div>
      <span id="statusText">All Systems Operational</span>
    </div>
    <span id="lastCheck" style="color: #888;">Last check: --:--</span>
  </div>

  <div class="summary-cards">
    <div class="summary-card">
      <div class="number" id="uptimeHours">0h</div>
      <div class="label">Agent Uptime</div>
    </div>
    <div class="summary-card">
      <div class="number" id="totalChecks">0</div>
      <div class="label">Health Checks</div>
    </div>
    <div class="summary-card">
      <div class="number" id="autoRestarts">0</div>
      <div class="label">Auto-Restarts Today</div>
    </div>
    <div class="summary-card">
      <div class="number" id="healthyProcesses">0</div>
      <div class="label">Healthy Processes</div>
    </div>
    <div class="summary-card">
      <div class="number" id="domainsUp">0</div>
      <div class="label">Domains Online</div>
    </div>
  </div>

  <div class="section">
    <h2>🌐 Live Domains Health</h2>
    <div id="domainsGrid">Loading...</div>
  </div>

  <div class="section">
    <h2>📊 Monitored Processes</h2>
    <div id="processesTable">Loading...</div>
  </div>

  <div class="section">
    <h2>📝 Recent Activity</h2>
    <div id="activityLog">Loading...</div>
  </div>

  <script>
    async function refreshStatus() {
      try {
        const data = await fetch('/api/status').then(r => r.json());

        // Update summary
        const uptimeHours = Math.floor((Date.now() - new Date(data.stats.uptime).getTime()) / (1000 * 60 * 60));
        document.getElementById('uptimeHours').textContent = uptimeHours + 'h';
        document.getElementById('totalChecks').textContent = data.stats.checksPerformed;
        document.getElementById('autoRestarts').textContent = data.stats.restartsToday.length;
        document.getElementById('healthyProcesses').textContent = data.processes.filter(p => p.status === 'online').length;
        document.getElementById('domainsUp').textContent = data.domains ? data.domains.filter(d => d.status === 'up').length + '/' + data.domains.length : '0/0';
        document.getElementById('lastCheck').textContent = 'Last check: ' + new Date(data.stats.lastCheck).toLocaleTimeString();

        // Domains grid
        const domainsDiv = document.getElementById('domainsGrid');
        if (data.domains && data.domains.length > 0) {
          domainsDiv.innerHTML = '<div class="domain-grid">' + data.domains.map(d => \`
            <div class="domain-card \${d.status === 'down' ? 'down' : ''}">
              <div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px;">
                <strong style="font-size: 0.95em;">\${d.domain}</strong>
                <span class="status-badge status-\${d.status}">\${d.status.toUpperCase()}</span>
              </div>
              \${d.responseTime ? \`<div style="color: #666; font-size: 0.85em;">Response: \${d.responseTime}ms</div>\` : ''}
              <div style="color: #888; font-size: 0.8em; margin-top: 5px;">Checked: \${new Date(d.lastCheck).toLocaleTimeString()}</div>
              <a href="https://\${d.domain}" target="_blank" style="color: #00C851; text-decoration: none; font-size: 0.85em; margin-top: 5px; display: inline-block;">🔗 Visit</a>
            </div>
          \`).join('') + '</div>';
        } else {
          domainsDiv.innerHTML = '<p style="color: #888;">Loading domains...</p>';
        }

        // Processes table
        const processesDiv = document.getElementById('processesTable');
        if (data.processes.length > 0) {
          processesDiv.innerHTML = \`
            <table class="process-table">
              <thead>
                <tr>
                  <th>Name</th>
                  <th>Status</th>
                  <th>Uptime</th>
                  <th>Restarts</th>
                  <th>Memory</th>
                </tr>
              </thead>
              <tbody>
                \${data.processes.map(p => \`
                  <tr>
                    <td><strong>\${p.name}</strong></td>
                    <td><span class="status-badge status-\${p.status}">\${p.status.toUpperCase()}</span></td>
                    <td>\${p.uptime}</td>
                    <td>\${p.restarts}</td>
                    <td>\${p.memory}</td>
                  </tr>
                \`).join('')}
              </tbody>
            </table>
          \`;
        } else {
          processesDiv.innerHTML = '<p style="color: #888;">No processes found</p>';
        }

        // Activity log
        const activityDiv = document.getElementById('activityLog');
        if (data.activity.length > 0) {
          activityDiv.innerHTML = data.activity.slice(0, 20).map(a => \`
            <div class="activity-item">
              <div class="action">\${a.action}: \${a.process}</div>
              <div>\${a.details}</div>
              <div class="time">\${new Date(a.timestamp).toLocaleTimeString()}</div>
            </div>
          \`).join('');
        } else {
          activityDiv.innerHTML = '<p style="color: #888;">No activity yet</p>';
        }
      } catch (error) {
        console.error('Error fetching status:', error);
      }
    }

    // Refresh every 5 seconds (monitor checks every 15 seconds)
    setInterval(refreshStatus, 5000);
    refreshStatus();
  </script>
</body>
</html>
  `);
});

// API endpoint for status
app.get('/api/status', requireAuth, async (req: Request, res: Response) => {
  try {
    const { stdout: pm2Output } = await execAsync('pm2 jlist');
    const pm2Processes = JSON.parse(pm2Output);

    const processes = pm2Processes.map((p: any) => ({
      id: p.pm_id,
      name: p.name,
      status: p.pm2_env.status,
      uptime: p.pm2_env.pm_uptime ? formatUptime(Date.now() - p.pm2_env.pm_uptime) : '0s',
      restarts: p.pm2_env.restart_time,
      memory: (p.monit.memory / 1024 / 1024).toFixed(1) + 'mb',
      pid: p.pid
    }));

    // Get domain status
    const domains = LIVE_DOMAINS.map(domain => {
      const health = domainHealth.get(domain) || { status: 'checking', lastCheck: new Date() };
      return {
        domain,
        status: health.status,
        lastCheck: health.lastCheck,
        responseTime: health.responseTime
      };
    });

    res.json({
      processes,
      domains,
      stats: {
        uptime: stats.uptime,
        checksPerformed: stats.checksPerformed,
        domainChecksPerformed: stats.domainChecksPerformed,
        restartsToday: stats.restartsToday,
        lastCheck: stats.lastCheck,
        lastDomainCheck: stats.lastDomainCheck
      },
      activity: activityLog
    });
  } catch (error) {
    console.error('Error fetching status:', error);
    res.status(500).json({ error: 'Failed to fetch status' });
  }
});

function formatUptime(ms: number): string {
  const seconds = Math.floor(ms / 1000);
  const minutes = Math.floor(seconds / 60);
  const hours = Math.floor(minutes / 60);
  const days = Math.floor(hours / 24);

  if (days > 0) return `${days}d`;
  if (hours > 0) return `${hours}h`;
  if (minutes > 0) return `${minutes}m`;
  return `${seconds}s`;
}

// Main monitoring loop - runs every 15 seconds
async function monitorProcesses() {
  try {
    stats.checksPerformed++;
    stats.lastCheck = new Date();

    const { stdout: pm2Output } = await execAsync('pm2 jlist');
    const pm2Processes = JSON.parse(pm2Output);

    for (const process of pm2Processes) {
      const name = process.name;
      const status = process.pm2_env.status;
      const restarts = process.pm2_env.restart_time;
      const uptime = process.pm2_env.pm_uptime;
      const uptimeSeconds = uptime ? (Date.now() - uptime) / 1000 : 0;

      // **CRASH LOOP DETECTION DISABLED** - Allowing processes to restart freely
      // if (restarts > 100 && uptimeSeconds < 60) {
      //   console.log(`🚨 CRASH LOOP DETECTED: ${name} has ${restarts} restarts and uptime ${uptimeSeconds}s`);
      //   console.log(`⚡ FORCING STOP AND DELAYED RESTART...`);
      //   try {
      //     await execAsync(`pm2 stop ${process.pm_id}`);
      //     console.log(`✋ Stopped ${name} to break crash loop`);
      //     await new Promise(resolve => setTimeout(resolve, 5000));
      //     await execAsync(`pm2 flush ${process.pm_id}`);
      //     await execAsync(`pm2 restart ${process.pm_id}`);
      //     stats.totalRestarts++;
      //     stats.restartsToday.push({
      //       name,
      //       time: new Date(),
      //       reason: `CRASH LOOP FIX - ${restarts} restarts in ${uptimeSeconds}s`
      //     });
      //     logActivity('CRASH LOOP FIX', name, `Detected ${restarts} restarts, forced stop and restart`);
      //     console.log(`✅ Fixed crash loop for ${name}`);
      //   } catch (error) {
      //     console.error(`❌ Failed to fix crash loop for ${name}:`, error);
      //     logActivity('CRASH LOOP FIX FAILED', name, `Failed to fix: ${error}`);
      //   }
      //   continue;
      // }

      // **EXCESSIVE RESTARTS** - More than 50 restarts is suspicious
      if (restarts > 50) {
        // Check if this is a known fixed issue
        const knownFix = knownFixes.fixes.find((fix: any) => fix.processName === name);

        if (knownFix && restarts === knownFix.restartCountAtFix) {
          // This is a known fixed issue with stable restart count
          console.log(`✅ ${name}: ${restarts} restarts (STABLE - Known fix applied on ${new Date(knownFix.fixedDate).toLocaleDateString()})`);
        } else if (knownFix && restarts > knownFix.restartCountAtFix) {
          // Restart count increased after fix - NEW PROBLEM!
          console.log(`🚨 ALERT: ${name} restart count increased from ${knownFix.restartCountAtFix} to ${restarts} - FIX MAY HAVE FAILED!`);
          logActivity('FIX REGRESSION', name, `Restart count increased from ${knownFix.restartCountAtFix} to ${restarts} after fix`);
        } else {
          // Unknown issue
          console.log(`⚠️  WARNING: ${name} has ${restarts} restarts - INVESTIGATING...`);
          logActivity('EXCESSIVE RESTARTS', name, `Process has ${restarts} restarts - may be unstable`);
        }
      }

      // Auto-restart stopped, errored, OR stopped processes - NO EXCEPTIONS
      if (status !== 'online' && status !== 'launching') {
        console.log(`🚨 CRITICAL: Process ${name} is ${status} - AUTO-RESTARTING IMMEDIATELY`);

        try {
          // Try restart first
          await execAsync(`pm2 restart ${process.pm_id}`);

          stats.totalRestarts++;
          stats.restartsToday.push({
            name,
            time: new Date(),
            reason: `Auto-restart due to ${status} status`
          });

          logActivity('AUTO-RESTART', name, `Process was ${status}, automatically restarted`);
          console.log(`✅ Successfully restarted ${name}`);
        } catch (error) {
          console.error(`❌ Failed to restart ${name}, trying full reset:`, error);

          // If restart fails, try delete and re-add
          try {
            await execAsync(`pm2 delete ${process.pm_id}`);
            await new Promise(resolve => setTimeout(resolve, 1000));
            // Note: Can't automatically re-add without knowing the original command
            logActivity('RESTART FAILED - DELETED', name, `Failed to restart, deleted process. Manual re-add required.`);
          } catch (deleteError) {
            logActivity('RESTART FAILED', name, `Failed to auto-restart: ${error}`);
          }
        }
      }

      // Check for excessive restarts (more than 10 in last hour)
      const recentRestarts = stats.restartsToday.filter(r =>
        r.name === name && (Date.now() - r.time.getTime()) < 3600000
      );

      if (recentRestarts.length > 10) {
        logActivity('HIGH RESTART COUNT', name, `${recentRestarts.length} restarts in last hour - may need investigation`);
      }
    }
  } catch (error) {
    console.error('Monitor error:', error);
    logActivity('MONITOR ERROR', 'Server Uptime', `Error during health check: ${error}`);
  }
}

// Nginx port configuration monitoring
async function checkNginxPorts() {
  try {
    console.log(`🔧 Checking nginx port configurations...`);

    for (const [domain, expectedPort] of Object.entries(NGINX_PORT_MAPPINGS)) {
      try {
        // Read the nginx config for this domain
        const configPath = `/etc/nginx/sites-enabled/${domain}`;
        const { stdout: configContent } = await execAsync(`cat ${configPath} 2>/dev/null || echo ""`);

        if (!configContent) {
          console.log(`⚠️  No nginx config found for ${domain}`);
          continue;
        }

        // Extract all proxy_pass ports from the config
        const proxyPassMatches = configContent.match(/proxy_pass\s+http:\/\/127\.0\.0\.1:(\d+)/g);

        if (!proxyPassMatches) {
          console.log(`⚠️  No proxy_pass found in config for ${domain}`);
          continue;
        }

        // Check if any proxy_pass has wrong port
        let hasWrongPort = false;
        const wrongPorts: number[] = [];

        for (const match of proxyPassMatches) {
          const portMatch = match.match(/:(\d+)/);
          if (portMatch) {
            const currentPort = parseInt(portMatch[1]);
            if (currentPort !== expectedPort) {
              hasWrongPort = true;
              wrongPorts.push(currentPort);
            }
          }
        }

        if (hasWrongPort) {
          console.log(`🚨 NGINX PORT MISMATCH DETECTED: ${domain}`);
          console.log(`   Expected port: ${expectedPort}`);
          console.log(`   Found ports: ${wrongPorts.join(', ')}`);
          console.log(`   AUTO-FIXING NOW...`);

          try {
            // Fix all wrong ports in the config
            for (const wrongPort of [...new Set(wrongPorts)]) {
              await execAsync(`sudo sed -i 's/:${wrongPort}/:${expectedPort}/g' ${configPath}`);
              console.log(`   ✅ Replaced ${wrongPort} → ${expectedPort}`);
            }

            // Test nginx config
            const { stdout: testResult } = await execAsync('sudo nginx -t 2>&1');
            console.log(`   Nginx config test: ${testResult.includes('successful') ? '✅ OK' : '⚠️  WARNING'}`);

            // Reload nginx
            await execAsync('sudo systemctl reload nginx');
            console.log(`   ✅ Nginx reloaded successfully`);

            logActivity('NGINX PORT FIX', domain, `Fixed port mismatch: ${wrongPorts.join(',')} → ${expectedPort}`);
          } catch (error) {
            console.error(`   ❌ Failed to fix nginx config for ${domain}:`, error);
            logActivity('NGINX FIX FAILED', domain, `Failed to fix port: ${error}`);
          }
        } else {
          console.log(`✅ ${domain} nginx config correct (port ${expectedPort})`);
        }
      } catch (error) {
        console.error(`Error checking nginx config for ${domain}:`, error);
      }
    }

    console.log(`✅ Nginx port check complete`);
  } catch (error) {
    console.error('Nginx monitoring error:', error);
    logActivity('NGINX MONITOR ERROR', 'Nginx', `Error during nginx check: ${error}`);
  }
}

// Domain health monitoring function
async function checkDomainHealth() {
  try {
    stats.domainChecksPerformed++;
    stats.lastDomainCheck = new Date();

    console.log(`🌐 Checking health of ${LIVE_DOMAINS.length} domains...`);

    // Check all domains in parallel
    const checks = LIVE_DOMAINS.map(async (domain) => {
      try {
        // Mark as checking
        domainHealth.set(domain, {
          status: 'checking',
          lastCheck: new Date()
        });

        const startTime = Date.now();

        // Try HTTPS first
        const url = `https://${domain}`;

        // Use fetch with timeout
        const controller = new AbortController();
        const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout

        const response = await fetch(url, {
          method: 'HEAD',
          signal: controller.signal,
          headers: {
            'User-Agent': 'DW-Server-Uptime-Agent/1.0'
          }
        });

        clearTimeout(timeoutId);

        const responseTime = Date.now() - startTime;

        // Consider 2xx, 3xx as success
        if (response.ok || (response.status >= 300 && response.status < 400)) {
          const previousStatus = domainHealth.get(domain)?.status;

          domainHealth.set(domain, {
            status: 'up',
            lastCheck: new Date(),
            responseTime
          });

          // Log if domain just came back up
          if (previousStatus === 'down') {
            console.log(`✅ Domain ${domain} is back UP (${responseTime}ms)`);
            logActivity('DOMAIN UP', domain, `Domain recovered - response time: ${responseTime}ms`);
          }
        } else {
          throw new Error(`HTTP ${response.status}`);
        }
      } catch (error) {
        const previousStatus = domainHealth.get(domain)?.status;

        domainHealth.set(domain, {
          status: 'down',
          lastCheck: new Date()
        });

        // Log domain down event
        if (previousStatus !== 'down') {
          console.log(`🚨 DOMAIN DOWN: ${domain} - ${error}`);
          logActivity('DOMAIN DOWN', domain, `Domain is not responding: ${error}`);
        }
      }
    });

    await Promise.all(checks);

    const upCount = Array.from(domainHealth.values()).filter(h => h.status === 'up').length;
    const downCount = Array.from(domainHealth.values()).filter(h => h.status === 'down').length;

    console.log(`✅ Domain check complete: ${upCount} up, ${downCount} down`);
  } catch (error) {
    console.error('Domain monitoring error:', error);
    logActivity('DOMAIN MONITOR ERROR', 'Domain Health', `Error during domain check: ${error}`);
  }
}

// Start monitoring loop - every 15 seconds
setInterval(monitorProcesses, 15000);

// Start domain health monitoring - every 60 seconds
setInterval(checkDomainHealth, 60000);

// Start nginx port monitoring - every 120 seconds (2 minutes)
setInterval(checkNginxPorts, 120000);

// Initial check on startup
setTimeout(() => {
  monitorProcesses();
  checkDomainHealth();
  checkNginxPorts();
  logActivity('STARTUP', 'Server Uptime', 'Monitoring agent started successfully');
}, 5000);


// Metrics endpoint for daily reporting
app.get("/api/metrics", (req: Request, res: Response) => {
  res.json({
    status: "online",
    uptime: process.uptime(),
    responseTime: 0,
    tasksCompleted: 0,
    errorsToday: 0,
    lastActivity: new Date().toISOString(),
    qnaReadiness: 100
  });
});

app.listen(PORT, '0.0.0.0', () => {
  console.log(`🟢 Server Uptime Agent running on port ${PORT}`);
  console.log(`Dashboard: http://45.61.58.125:${PORT}`);
  console.log(`⚡ Monitoring all processes every 15 seconds`);
  console.log(`⚡ Auto-restart enabled for all stopped/errored processes`);
});