← back to Designer Wallcoverings
DW-Agents/dw-agents/agent-server-uptime/server-uptime-agent.ts
1120 lines
/**
* DW-Agents: Server Uptime Agent
*
* Monitors all processes and ensures they never crash
* - Checks all PM2 processes every 15 seconds
* - Auto-restarts any stopped/errored processes immediately
* - Monitors process health (CPU, memory, restart counts)
* - Alerts on critical issues
* - Ensures 100% uptime across all agents
*
* Port: 9887
*/
import express, { Request, Response } from 'express';
import { requireAuth as ssoAuth, handleLogin, setSSOToken, SSO_TOKEN_NAME, SSO_TOKEN_VALUE } from './shared-auth';
import cookieParser from 'cookie-parser';
import session from 'express-session';
import { exec } from 'child_process';
import { promisify } from 'util';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
// Fix __dirname for ES modules
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const execAsync = promisify(exec);
// Load known fixes
const KNOWN_FIXES_FILE = path.join(__dirname, 'known-fixes.json');
let knownFixes: any = { fixes: [] };
try {
if (fs.existsSync(KNOWN_FIXES_FILE)) {
knownFixes = JSON.parse(fs.readFileSync(KNOWN_FIXES_FILE, 'utf8'));
}
} catch (error) {
console.error('Failed to load known-fixes.json:', error);
}
const app = express();
const PORT = process.env.PORT ? parseInt(process.env.PORT) : 9882;
app.use(express.json());
app.use(cookieParser());
app.use(express.urlencoded({ extended: true }));
declare module 'express-session' {
interface SessionData {
authenticated: boolean;
}
}
app.use(
session({
secret: 'dw-server-uptime-2025',
resave: false,
saveUninitialized: true,
cookie: {
secure: false,
httpOnly: true,
maxAge: 7 * 24 * 60 * 60 * 1000,
},
})
);
const requireAuth = (req: Request, res: Response, next: any) => {
// Check session first
if (req.session.authenticated) {
return next();
}
// Check SSO cookie
if (req.cookies && req.cookies[SSO_TOKEN_NAME] === SSO_TOKEN_VALUE) {
req.session.authenticated = true;
return next();
}
res.redirect('/login');
};
// Live domains to monitor with their expected backend ports
const LIVE_DOMAINS = [
'cypresawards.com',
'flockedwallpaper.com',
'glassbeadedwallpaper.com',
'goodquestion.ai',
'grassclothwallcoverings.com',
'grassclothwallpaper.com',
'latinogardens.greendomainbrokers.com',
'losgardeners.com',
'n8n.greendomainbrokers.com',
'novasuede.com',
'www.cypresawards.com',
'www.flockedwallpaper.com',
'www.glassbeadedwallpaper.com',
'www.goodquestion.ai',
'www.grassclothwallcoverings.com',
'www.grassclothwallpaper.com',
'www.losgardeners.com',
'www.novasuede.com'
];
// Expected nginx port mappings (domain -> correct backend port)
const NGINX_PORT_MAPPINGS: { [key: string]: number } = {
'grassclothwallpaper.com': 3201,
'www.grassclothwallpaper.com': 3201,
'flockedwallpaper.com': 3200,
'www.flockedwallpaper.com': 3200,
'novasuede.com': 3204,
'www.novasuede.com': 3204,
'glassbeadedwallpaper.com': 3203,
'www.glassbeadedwallpaper.com': 3203
};
// Stats tracking
const stats = {
totalRestarts: 0,
lastCheck: new Date(),
lastDomainCheck: new Date(),
restartsToday: [] as Array<{ name: string; time: Date; reason: string }>,
uptime: Date.now(),
checksPerformed: 0,
domainChecksPerformed: 0,
errors: [] as Array<{ timestamp: Date; error: string; recovered: boolean }>,
recoveryActions: [] as Array<{ timestamp: Date; action: string; success: boolean }>
};
// Domain health tracking
const domainHealth: Map<string, { status: 'up' | 'down' | 'checking'; lastCheck: Date; responseTime?: number }> = new Map();
// Activity log
const activityLog: Array<{ timestamp: Date; action: string; process: string; details: string }> = [];
function logActivity(action: string, process: string, details: string) {
try {
activityLog.unshift({
timestamp: new Date(),
action,
process,
details
});
if (activityLog.length > 100) activityLog.pop();
// Also log to console with timestamp
const timestamp = new Date().toISOString();
console.log(`[${timestamp}] ${action}: ${process} - ${details}`);
} catch (error) {
console.error('Error logging activity:', error);
}
}
function logError(error: string, recovered: boolean = false) {
try {
stats.errors.unshift({
timestamp: new Date(),
error,
recovered
});
if (stats.errors.length > 50) stats.errors.pop();
} catch (e) {
console.error('Error logging error:', e);
}
}
function logRecoveryAction(action: string, success: boolean) {
try {
stats.recoveryActions.unshift({
timestamp: new Date(),
action,
success
});
if (stats.recoveryActions.length > 50) stats.recoveryActions.pop();
} catch (e) {
console.error('Error logging recovery action:', e);
}
}
// Login page
app.get('/login', (req: Request, res: Response) => {
res.send(`
<!DOCTYPE html>
<html>
<head>
<title>Server Uptime - Login</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: linear-gradient(135deg, #00C851 0%, #007E33 100%);
display: flex;
align-items: center;
justify-content: center;
min-height: 100vh;
}
.login-container {
background: white;
padding: 40px;
border-radius: 15px;
box-shadow: 0 15px 35px rgba(0,0,0,0.2);
max-width: 400px;
width: 100%;
}
h1 { color: #00C851; text-align: center; margin-bottom: 30px; }
input {
width: 100%;
padding: 12px;
margin: 10px 0;
border: 2px solid #e0e0e0;
border-radius: 8px;
}
button {
width: 100%;
background: linear-gradient(135deg, #00C851 0%, #007E33 100%);
color: white;
border: none;
padding: 15px;
border-radius: 8px;
cursor: pointer;
font-weight: 600;
margin-top: 10px;
}
</style>
</head>
<body>
<div class="login-container">
<h1>🟢 Server Uptime</h1>
<form method="POST">
<input type="text" name="username" placeholder="Username" required>
<input type="password" name="password" placeholder="Password" required>
<button type="submit">Login</button>
</form>
</div>
</body>
</html>
`);
});
app.post('/login', (req: Request, res: Response) => {
const { username, password } = req.body;
if (username === 'admin' && password === '2025') {
req.session.authenticated = true;
// Set SSO cookie for cross-agent authentication
setSSOToken(res);
res.redirect('/');
} else {
res.redirect('/login');
}
});
// Main dashboard
app.get('/', requireAuth, (req: Request, res: Response) => {
res.send(`
<!DOCTYPE html>
<html>
<head>
<title>Server Uptime Agent - DW-Agents</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: linear-gradient(135deg, #00C851 0%, #007E33 100%);
padding: 20px;
min-height: 100vh;
}
.header {
background: white;
padding: 30px;
border-radius: 15px;
margin-bottom: 20px;
box-shadow: 0 5px 20px rgba(0,0,0,0.1);
}
.header h1 { color: #00C851; font-size: 2.5em; margin-bottom: 10px; }
.header .subtitle { color: #666; font-size: 1.1em; }
.status-bar {
background: white;
padding: 15px 30px;
border-radius: 15px;
margin-bottom: 20px;
display: flex;
justify-content: space-between;
align-items: center;
box-shadow: 0 5px 20px rgba(0,0,0,0.1);
}
.status-indicator {
display: flex;
align-items: center;
gap: 10px;
font-weight: 600;
}
.status-indicator.healthy { color: #00C851; }
.pulse {
width: 12px;
height: 12px;
background: #00C851;
border-radius: 50%;
animation: pulse 2s infinite;
}
@keyframes pulse {
0%, 100% { opacity: 1; transform: scale(1); }
50% { opacity: 0.5; transform: scale(1.2); }
}
.summary-cards {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 15px;
margin-bottom: 20px;
}
.summary-card {
background: white;
padding: 20px;
border-radius: 12px;
box-shadow: 0 5px 20px rgba(0,0,0,0.1);
text-align: center;
}
.summary-card .number {
font-size: 2.5em;
font-weight: bold;
color: #00C851;
}
.summary-card .label {
color: #666;
margin-top: 5px;
}
.section {
background: white;
padding: 25px;
border-radius: 15px;
margin-bottom: 20px;
box-shadow: 0 5px 20px rgba(0,0,0,0.1);
}
.section h2 {
color: #00C851;
margin-bottom: 15px;
font-size: 1.5em;
}
.process-table {
width: 100%;
border-collapse: collapse;
font-size: 0.9em;
}
.process-table th {
background: #f8f9fa;
padding: 12px;
text-align: left;
font-weight: 600;
border-bottom: 2px solid #e0e0e0;
}
.process-table td {
padding: 12px;
border-bottom: 1px solid #e0e0e0;
}
.status-badge {
display: inline-block;
padding: 4px 12px;
border-radius: 12px;
font-size: 0.85em;
font-weight: 600;
}
.status-online { background: #d1fae5; color: #065f46; }
.status-stopped { background: #fee2e2; color: #991b1b; }
.status-errored { background: #fef3c7; color: #92400e; }
.status-up { background: #d1fae5; color: #065f46; }
.status-down { background: #fee2e2; color: #991b1b; }
.domain-grid {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(250px, 1fr));
gap: 15px;
margin-top: 15px;
}
.domain-card {
background: #f8f9fa;
padding: 15px;
border-radius: 8px;
border-left: 4px solid #00C851;
}
.domain-card.down {
border-left-color: #dc143c;
background: #fff5f5;
}
.activity-item {
background: #f8f9fa;
padding: 12px;
border-radius: 8px;
margin-bottom: 8px;
border-left: 4px solid #00C851;
}
.activity-item .time { color: #888; font-size: 0.85em; }
.activity-item .action { font-weight: 600; color: #333; }
</style>
</head>
<body>
<div class="header">
<h1>🟢 Server Uptime Agent</h1>
<div class="subtitle">Automatic process monitoring and recovery - Checks every 5 minutes</div>
<div class="subtitle" id="countdown-display" style="margin-top: 10px; color: #00C851; font-weight: 600;"></div>
</div>
<div class="status-bar">
<div class="status-indicator healthy">
<div class="pulse"></div>
<span id="statusText">All Systems Operational</span>
</div>
<span id="lastCheck" style="color: #888;">Last check: --:--</span>
</div>
<div class="summary-cards">
<div class="summary-card">
<div class="number" id="uptimeHours">0h</div>
<div class="label">Agent Uptime</div>
</div>
<div class="summary-card">
<div class="number" id="totalChecks">0</div>
<div class="label">Health Checks</div>
</div>
<div class="summary-card">
<div class="number" id="autoRestarts">0</div>
<div class="label">Auto-Restarts Today</div>
</div>
<div class="summary-card">
<div class="number" id="healthyProcesses">0</div>
<div class="label">Healthy Processes</div>
</div>
<div class="summary-card">
<div class="number" id="domainsUp">0</div>
<div class="label">Domains Online</div>
</div>
</div>
<div class="section">
<h2>🌐 Live Domains Health</h2>
<div id="domainsGrid">Loading...</div>
</div>
<div class="section">
<h2>📊 Monitored Processes</h2>
<div id="processesTable">Loading...</div>
</div>
<div class="section">
<h2>🔧 Auto-Recovery Actions</h2>
<div id="recoveryLog">Loading...</div>
</div>
<div class="section">
<h2>⚠️ Recent Errors</h2>
<div id="errorLog">Loading...</div>
</div>
<div class="section">
<h2>📝 Recent Activity</h2>
<div id="activityLog">Loading...</div>
</div>
<script>
async function refreshStatus() {
try {
const data = await fetch('/api/status').then(r => r.json());
// Update summary
const uptimeHours = Math.floor((Date.now() - new Date(data.stats.uptime).getTime()) / (1000 * 60 * 60));
document.getElementById('uptimeHours').textContent = uptimeHours + 'h';
document.getElementById('totalChecks').textContent = data.stats.checksPerformed;
document.getElementById('autoRestarts').textContent = data.stats.restartsToday.length;
document.getElementById('healthyProcesses').textContent = data.processes.filter(p => p.status === 'online').length;
document.getElementById('domainsUp').textContent = data.domains ? data.domains.filter(d => d.status === 'up').length + '/' + data.domains.length : '0/0';
document.getElementById('lastCheck').textContent = 'Last check: ' + new Date(data.stats.lastCheck).toLocaleTimeString();
// Update countdown display
updateCountdown(data.stats.nextCheckIn);
// Domains grid
const domainsDiv = document.getElementById('domainsGrid');
if (data.domains && data.domains.length > 0) {
domainsDiv.innerHTML = '<div class="domain-grid">' + data.domains.map(d => \`
<div class="domain-card \${d.status === 'down' ? 'down' : ''}">
<div style="display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px;">
<strong style="font-size: 0.95em;">\${d.domain}</strong>
<span class="status-badge status-\${d.status}">\${d.status.toUpperCase()}</span>
</div>
\${d.responseTime ? \`<div style="color: #666; font-size: 0.85em;">Response: \${d.responseTime}ms</div>\` : ''}
<div style="color: #888; font-size: 0.8em; margin-top: 5px;">Checked: \${new Date(d.lastCheck).toLocaleTimeString()}</div>
<a href="https://\${d.domain}" target="_blank" style="color: #00C851; text-decoration: none; font-size: 0.85em; margin-top: 5px; display: inline-block;">🔗 Visit</a>
</div>
\`).join('') + '</div>';
} else {
domainsDiv.innerHTML = '<p style="color: #888;">Loading domains...</p>';
}
// Processes table
const processesDiv = document.getElementById('processesTable');
if (data.processes.length > 0) {
processesDiv.innerHTML = \`
<table class="process-table">
<thead>
<tr>
<th>Name</th>
<th>Status</th>
<th>Uptime</th>
<th>Restarts</th>
<th>Memory</th>
</tr>
</thead>
<tbody>
\${data.processes.map(p => \`
<tr>
<td><strong>\${p.name}</strong></td>
<td><span class="status-badge status-\${p.status}">\${p.status.toUpperCase()}</span></td>
<td>\${p.uptime}</td>
<td>\${p.restarts}</td>
<td>\${p.memory}</td>
</tr>
\`).join('')}
</tbody>
</table>
\`;
} else {
processesDiv.innerHTML = '<p style="color: #888;">No processes found</p>';
}
// Recovery actions log
const recoveryDiv = document.getElementById('recoveryLog');
if (data.stats.recoveryActions && data.stats.recoveryActions.length > 0) {
recoveryDiv.innerHTML = data.stats.recoveryActions.slice(0, 10).map(r => \`
<div class="activity-item" style="border-left-color: \${r.success ? '#00C851' : '#dc143c'};">
<div class="action">\${r.success ? '✅' : '❌'} \${r.action}</div>
<div class="time">\${new Date(r.timestamp).toLocaleTimeString()}</div>
</div>
\`).join('');
} else {
recoveryDiv.innerHTML = '<p style="color: #888;">No recovery actions yet</p>';
}
// Error log
const errorDiv = document.getElementById('errorLog');
if (data.stats.errors && data.stats.errors.length > 0) {
errorDiv.innerHTML = data.stats.errors.slice(0, 10).map(e => \`
<div class="activity-item" style="border-left-color: \${e.recovered ? '#f59e0b' : '#dc143c'}; background: \${e.recovered ? '#fffbeb' : '#fff5f5'};">
<div class="action">\${e.recovered ? '🔧 Recovered' : '❌ Error'}</div>
<div>\${e.error}</div>
<div class="time">\${new Date(e.timestamp).toLocaleTimeString()}</div>
</div>
\`).join('');
} else {
errorDiv.innerHTML = '<p style="color: #888;">No errors - system healthy!</p>';
}
// Activity log
const activityDiv = document.getElementById('activityLog');
if (data.activity.length > 0) {
activityDiv.innerHTML = data.activity.slice(0, 20).map(a => \`
<div class="activity-item">
<div class="action">\${a.action}: \${a.process}</div>
<div>\${a.details}</div>
<div class="time">\${new Date(a.timestamp).toLocaleTimeString()}</div>
</div>
\`).join('');
} else {
activityDiv.innerHTML = '<p style="color: #888;">No activity yet</p>';
}
} catch (error) {
console.error('Error fetching status:', error);
}
}
// Update countdown display
function updateCountdown(secondsRemaining) {
const countdownEl = document.getElementById('countdown-display');
if (countdownEl && secondsRemaining !== undefined) {
const minutes = Math.floor(secondsRemaining / 60);
const seconds = secondsRemaining % 60;
countdownEl.textContent = 'Next server check in: ' + minutes + 'm ' + seconds + 's';
}
}
// Refresh every 5 seconds (monitor checks every 5 minutes)
setInterval(refreshStatus, 5000);
refreshStatus();
</script>
</body>
</html>
`);
});
// API endpoint for status
app.get('/api/status', requireAuth, async (req: Request, res: Response) => {
try {
const { stdout: pm2Output } = await execAsync('pm2 jlist');
const pm2Processes = JSON.parse(pm2Output);
const processes = pm2Processes.map((p: any) => ({
id: p.pm_id,
name: p.name,
status: p.pm2_env.status,
uptime: p.pm2_env.pm_uptime ? formatUptime(Date.now() - p.pm2_env.pm_uptime) : '0s',
restarts: p.pm2_env.restart_time,
memory: (p.monit.memory / 1024 / 1024).toFixed(1) + 'mb',
pid: p.pid
}));
// Get domain status
const domains = LIVE_DOMAINS.map(domain => {
const health = domainHealth.get(domain) || { status: 'checking', lastCheck: new Date() };
return {
domain,
status: health.status,
lastCheck: health.lastCheck,
responseTime: health.responseTime
};
});
res.json({
processes,
domains,
stats: {
uptime: stats.uptime,
checksPerformed: stats.checksPerformed,
domainChecksPerformed: stats.domainChecksPerformed,
restartsToday: stats.restartsToday,
lastCheck: stats.lastCheck,
lastDomainCheck: stats.lastDomainCheck,
errors: stats.errors,
recoveryActions: stats.recoveryActions,
nextCheckIn: Math.max(0, Math.floor((nextCheckTime - Date.now()) / 1000)) // seconds until next check
},
activity: activityLog
});
} catch (error) {
console.error('Error fetching status:', error);
res.status(500).json({ error: 'Failed to fetch status' });
}
});
function formatUptime(ms: number): string {
const seconds = Math.floor(ms / 1000);
const minutes = Math.floor(seconds / 60);
const hours = Math.floor(minutes / 60);
const days = Math.floor(hours / 24);
if (days > 0) return `${days}d`;
if (hours > 0) return `${hours}h`;
if (minutes > 0) return `${minutes}m`;
return `${seconds}s`;
}
// Main monitoring loop - runs every 15 seconds
async function monitorProcesses() {
const checkStartTime = Date.now();
try {
stats.checksPerformed++;
stats.lastCheck = new Date();
// Try to get PM2 process list with timeout
let pm2Output: string;
try {
const result = await execAsync('pm2 jlist', { timeout: 5000 });
pm2Output = result.stdout;
} catch (execError: any) {
const errorMsg = `Failed to execute pm2 jlist: ${execError.message}`;
console.error(`❌ ${errorMsg}`);
logError(errorMsg, false);
logRecoveryAction('Retry pm2 jlist', false);
// Retry once after 2 seconds
await new Promise(resolve => setTimeout(resolve, 2000));
try {
const result = await execAsync('pm2 jlist', { timeout: 5000 });
pm2Output = result.stdout;
logRecoveryAction('Retry pm2 jlist', true);
} catch (retryError: any) {
logError(`PM2 jlist retry failed: ${retryError.message}`, false);
return; // Skip this check cycle
}
}
// Parse PM2 output
let pm2Processes: any[];
try {
// Clean the output - remove any leading/trailing whitespace or special characters
const cleanOutput = pm2Output.trim();
// Handle empty output
if (!cleanOutput) {
console.warn('Empty PM2 output received');
pm2Processes = [];
} else if (!cleanOutput.startsWith('[') && !cleanOutput.startsWith('{')) {
// Try to extract JSON from output that may have extra text
const jsonMatch = cleanOutput.match(/(\[[\s\S]*\]|\{[\s\S]*\})/);
if (jsonMatch) {
pm2Processes = JSON.parse(jsonMatch[1]);
} else {
console.warn('Invalid PM2 output format, using empty array');
pm2Processes = [];
}
} else {
pm2Processes = JSON.parse(cleanOutput);
}
} catch (parseError: any) {
const errorMsg = `Failed to parse PM2 output: ${parseError.message}`;
console.error(`❌ ${errorMsg}`);
logError(errorMsg, false);
return; // Skip this check cycle
}
for (const process of pm2Processes) {
const name = process.name;
const status = process.pm2_env.status;
const restarts = process.pm2_env.restart_time;
const uptime = process.pm2_env.pm_uptime;
const uptimeSeconds = uptime ? (Date.now() - uptime) / 1000 : 0;
// **CRASH LOOP DETECTION** - Process restarting too many times
// Skip crash loop detection for the server-uptime agent itself
if (name !== 'dw-server-uptime' && restarts > 100 && uptimeSeconds < 60) {
console.log(`🚨 CRASH LOOP DETECTED: ${name} has ${restarts} restarts and uptime ${uptimeSeconds}s`);
console.log(`⚡ FORCING STOP AND DELAYED RESTART...`);
try {
// Stop the process completely
await execAsync(`pm2 stop ${process.pm_id}`);
console.log(`✋ Stopped ${name} to break crash loop`);
// Wait 5 seconds
await new Promise(resolve => setTimeout(resolve, 5000));
// Delete and flush logs to clear any bad state
await execAsync(`pm2 flush ${process.pm_id}`);
// Restart fresh
await execAsync(`pm2 restart ${process.pm_id}`);
stats.totalRestarts++;
stats.restartsToday.push({
name,
time: new Date(),
reason: `CRASH LOOP FIX - ${restarts} restarts in ${uptimeSeconds}s`
});
logActivity('CRASH LOOP FIX', name, `Detected ${restarts} restarts, forced stop and restart`);
console.log(`✅ Fixed crash loop for ${name}`);
} catch (error) {
console.error(`❌ Failed to fix crash loop for ${name}:`, error);
logActivity('CRASH LOOP FIX FAILED', name, `Failed to fix: ${error}`);
}
continue;
}
// **EXCESSIVE RESTARTS** - More than 50 restarts is suspicious
// Skip excessive restart warnings for the server-uptime agent itself
if (name !== 'dw-server-uptime' && restarts > 50) {
// Check if this is a known fixed issue
const knownFix = knownFixes.fixes.find((fix: any) => fix.processName === name);
if (knownFix && restarts === knownFix.restartCountAtFix) {
// This is a known fixed issue with stable restart count
console.log(`✅ ${name}: ${restarts} restarts (STABLE - Known fix applied on ${new Date(knownFix.fixedDate).toLocaleDateString()})`);
} else if (knownFix && restarts > knownFix.restartCountAtFix) {
// Restart count increased after fix - NEW PROBLEM!
console.log(`🚨 ALERT: ${name} restart count increased from ${knownFix.restartCountAtFix} to ${restarts} - FIX MAY HAVE FAILED!`);
logActivity('FIX REGRESSION', name, `Restart count increased from ${knownFix.restartCountAtFix} to ${restarts} after fix`);
} else {
// Unknown issue
console.log(`⚠️ WARNING: ${name} has ${restarts} restarts - INVESTIGATING...`);
logActivity('EXCESSIVE RESTARTS', name, `Process has ${restarts} restarts - may be unstable`);
}
}
// Auto-restart stopped, errored, OR stopped processes - NO EXCEPTIONS
if (status !== 'online' && status !== 'launching') {
const restartStartTime = Date.now();
console.log(`🚨 CRITICAL: Process ${name} is ${status} - AUTO-RESTARTING IMMEDIATELY`);
try {
// Try restart first (with timeout)
await execAsync(`pm2 restart ${process.pm_id}`, { timeout: 10000 });
const restartTime = Date.now() - restartStartTime;
stats.totalRestarts++;
stats.restartsToday.push({
name,
time: new Date(),
reason: `Auto-restart due to ${status} status`
});
logActivity('AUTO-RESTART', name, `Process was ${status}, automatically restarted in ${restartTime}ms`);
logRecoveryAction(`Restart ${name}`, true);
logError(`Process ${name} was ${status}`, true);
console.log(`✅ Successfully restarted ${name} in ${restartTime}ms`);
} catch (error: any) {
const errorMsg = `Failed to restart ${name}: ${error.message}`;
console.error(`❌ ${errorMsg}`);
logError(errorMsg, false);
logRecoveryAction(`Restart ${name}`, false);
// If restart fails, try multiple recovery strategies
console.log(`🔧 Trying alternative restart methods for ${name}...`);
// Strategy 1: Force restart with SIGINT
try {
console.log(` Strategy 1: Force restart with SIGINT...`);
await execAsync(`pm2 reload ${process.pm_id} --force`, { timeout: 10000 });
logActivity('FORCE RESTART', name, `Force restarted after normal restart failed`);
logRecoveryAction(`Force restart ${name}`, true);
logError(errorMsg, true);
console.log(`✅ Force restart successful for ${name}`);
continue;
} catch (forceError) {
console.error(` ❌ Force restart failed`);
}
// Strategy 2: Stop and start
try {
console.log(` Strategy 2: Stop and start...`);
await execAsync(`pm2 stop ${process.pm_id}`, { timeout: 5000 });
await new Promise(resolve => setTimeout(resolve, 2000));
await execAsync(`pm2 start ${process.pm_id}`, { timeout: 10000 });
logActivity('STOP-START RESTART', name, `Stop-start restart after normal restart failed`);
logRecoveryAction(`Stop-start ${name}`, true);
logError(errorMsg, true);
console.log(`✅ Stop-start successful for ${name}`);
continue;
} catch (stopStartError) {
console.error(` ❌ Stop-start failed`);
}
// Strategy 3: Delete and re-add (last resort)
try {
console.log(` Strategy 3: Delete and re-add (last resort)...`);
await execAsync(`pm2 delete ${process.pm_id}`, { timeout: 5000 });
await new Promise(resolve => setTimeout(resolve, 1000));
logActivity('RESTART FAILED - DELETED', name, `All restart strategies failed, deleted process. Manual intervention required.`);
logRecoveryAction(`Delete ${name}`, true);
console.log(`⚠️ Deleted ${name} - manual re-add required`);
} catch (deleteError: any) {
logActivity('RESTART FAILED - CRITICAL', name, `All recovery strategies failed: ${deleteError.message}`);
logRecoveryAction(`Delete ${name}`, false);
console.error(`❌ CRITICAL: All recovery strategies failed for ${name}`);
}
}
}
// Check for excessive restarts (more than 10 in last hour)
const recentRestarts = stats.restartsToday.filter(r =>
r.name === name && (Date.now() - r.time.getTime()) < 3600000
);
if (recentRestarts.length > 10) {
logActivity('HIGH RESTART COUNT', name, `${recentRestarts.length} restarts in last hour - may need investigation`);
}
}
const checkDuration = Date.now() - checkStartTime;
console.log(`✅ Health check completed in ${checkDuration}ms`);
} catch (error: any) {
const checkDuration = Date.now() - checkStartTime;
const errorMsg = `Monitor error after ${checkDuration}ms: ${error.message}`;
console.error(`❌ ${errorMsg}`);
console.error('Stack trace:', error.stack);
logActivity('MONITOR ERROR', 'Server Uptime', errorMsg);
logError(errorMsg, false);
// Try to recover by ensuring the monitoring continues
try {
console.log('🔧 Attempting to recover monitoring loop...');
// The setInterval will call this function again, so we just need to not crash
logRecoveryAction('Monitor loop recovery', true);
} catch (recoveryError: any) {
console.error('❌ Recovery failed:', recoveryError.message);
logRecoveryAction('Monitor loop recovery', false);
}
}
}
// Wrapper function to ensure monitoring never stops
async function monitorProcessesSafe() {
try {
await monitorProcesses();
} catch (error: any) {
console.error('❌ CRITICAL: Monitor process wrapper caught error:', error.message);
// Log but don't throw - ensure monitoring continues
logError(`Monitor wrapper error: ${error.message}`, false);
}
}
// Nginx port configuration monitoring
async function checkNginxPorts() {
try {
console.log(`🔧 Checking nginx port configurations...`);
for (const [domain, expectedPort] of Object.entries(NGINX_PORT_MAPPINGS)) {
try {
// Read the nginx config for this domain
const configPath = `/etc/nginx/sites-enabled/${domain}`;
const { stdout: configContent } = await execAsync(`cat ${configPath} 2>/dev/null || echo ""`);
if (!configContent) {
console.log(`⚠️ No nginx config found for ${domain}`);
continue;
}
// Extract all proxy_pass ports from the config
const proxyPassMatches = configContent.match(/proxy_pass\s+http:\/\/127\.0\.0\.1:(\d+)/g);
if (!proxyPassMatches) {
console.log(`⚠️ No proxy_pass found in config for ${domain}`);
continue;
}
// Check if any proxy_pass has wrong port
let hasWrongPort = false;
const wrongPorts: number[] = [];
for (const match of proxyPassMatches) {
const portMatch = match.match(/:(\d+)/);
if (portMatch) {
const currentPort = parseInt(portMatch[1]);
if (currentPort !== expectedPort) {
hasWrongPort = true;
wrongPorts.push(currentPort);
}
}
}
if (hasWrongPort) {
console.log(`🚨 NGINX PORT MISMATCH DETECTED: ${domain}`);
console.log(` Expected port: ${expectedPort}`);
console.log(` Found ports: ${wrongPorts.join(', ')}`);
console.log(` AUTO-FIXING NOW...`);
try {
// Fix all wrong ports in the config
for (const wrongPort of [...new Set(wrongPorts)]) {
await execAsync(`sudo sed -i 's/:${wrongPort}/:${expectedPort}/g' ${configPath}`);
console.log(` ✅ Replaced ${wrongPort} → ${expectedPort}`);
}
// Test nginx config
const { stdout: testResult } = await execAsync('sudo nginx -t 2>&1');
console.log(` Nginx config test: ${testResult.includes('successful') ? '✅ OK' : '⚠️ WARNING'}`);
// Reload nginx
await execAsync('sudo systemctl reload nginx');
console.log(` ✅ Nginx reloaded successfully`);
logActivity('NGINX PORT FIX', domain, `Fixed port mismatch: ${wrongPorts.join(',')} → ${expectedPort}`);
} catch (error) {
console.error(` ❌ Failed to fix nginx config for ${domain}:`, error);
logActivity('NGINX FIX FAILED', domain, `Failed to fix port: ${error}`);
}
} else {
console.log(`✅ ${domain} nginx config correct (port ${expectedPort})`);
}
} catch (error) {
console.error(`Error checking nginx config for ${domain}:`, error);
}
}
console.log(`✅ Nginx port check complete`);
} catch (error) {
console.error('Nginx monitoring error:', error);
logActivity('NGINX MONITOR ERROR', 'Nginx', `Error during nginx check: ${error}`);
}
}
// Domain health monitoring function
async function checkDomainHealth() {
const checkStartTime = Date.now();
try {
stats.domainChecksPerformed++;
stats.lastDomainCheck = new Date();
console.log(`🌐 Checking health of ${LIVE_DOMAINS.length} domains...`);
// Check all domains in parallel with individual error handling
const checks = LIVE_DOMAINS.map(async (domain) => {
try {
// Mark as checking
domainHealth.set(domain, {
status: 'checking',
lastCheck: new Date()
});
const startTime = Date.now();
// Try HTTPS first
const url = `https://${domain}`;
// Use fetch with timeout
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
const response = await fetch(url, {
method: 'HEAD',
signal: controller.signal,
headers: {
'User-Agent': 'DW-Server-Uptime-Agent/1.0'
}
});
clearTimeout(timeoutId);
const responseTime = Date.now() - startTime;
// Consider 2xx, 3xx as success
if (response.ok || (response.status >= 300 && response.status < 400)) {
const previousStatus = domainHealth.get(domain)?.status;
domainHealth.set(domain, {
status: 'up',
lastCheck: new Date(),
responseTime
});
// Log if domain just came back up
if (previousStatus === 'down') {
console.log(`✅ Domain ${domain} is back UP (${responseTime}ms)`);
logActivity('DOMAIN UP', domain, `Domain recovered - response time: ${responseTime}ms`);
}
} else {
throw new Error(`HTTP ${response.status}`);
}
} catch (error) {
const previousStatus = domainHealth.get(domain)?.status;
domainHealth.set(domain, {
status: 'down',
lastCheck: new Date()
});
// Log domain down event
if (previousStatus !== 'down') {
console.log(`🚨 DOMAIN DOWN: ${domain} - ${error}`);
logActivity('DOMAIN DOWN', domain, `Domain is not responding: ${error}`);
}
}
});
await Promise.allSettled(checks); // Use allSettled to handle individual failures
const upCount = Array.from(domainHealth.values()).filter(h => h.status === 'up').length;
const downCount = Array.from(domainHealth.values()).filter(h => h.status === 'down').length;
const checkDuration = Date.now() - checkStartTime;
console.log(`✅ Domain check complete in ${checkDuration}ms: ${upCount} up, ${downCount} down`);
} catch (error: any) {
const checkDuration = Date.now() - checkStartTime;
const errorMsg = `Domain monitoring error after ${checkDuration}ms: ${error.message}`;
console.error(`❌ ${errorMsg}`);
logActivity('DOMAIN MONITOR ERROR', 'Domain Health', errorMsg);
logError(errorMsg, false);
}
}
// Safe wrapper for domain health checking
async function checkDomainHealthSafe() {
try {
await checkDomainHealth();
} catch (error: any) {
console.error('❌ CRITICAL: Domain health wrapper caught error:', error.message);
logError(`Domain health wrapper error: ${error.message}`, false);
}
}
// Safe wrapper for nginx port checking
async function checkNginxPortsSafe() {
try {
await checkNginxPorts();
} catch (error: any) {
console.error('❌ CRITICAL: Nginx check wrapper caught error:', error.message);
logError(`Nginx check wrapper error: ${error.message}`, false);
}
}
// Start monitoring loops with safe wrappers - ensures they never crash
console.log('⚡ Initializing monitoring loops...');
// Track next check time for countdown
let nextCheckTime = Date.now() + 300000; // 5 minutes from now
// Process monitoring - every 5 minutes
setInterval(() => {
monitorProcessesSafe();
nextCheckTime = Date.now() + 300000; // Reset countdown
}, 300000);
console.log('✅ Process monitoring: every 5 minutes');
// Run first check immediately
monitorProcessesSafe();
// Domain health monitoring - every 60 seconds
setInterval(checkDomainHealthSafe, 60000);
console.log('✅ Domain health monitoring: every 60 seconds');
// Nginx port monitoring - every 120 seconds (2 minutes)
setInterval(checkNginxPortsSafe, 120000);
console.log('✅ Nginx port monitoring: every 120 seconds');
// Initial check on startup (delayed to allow server to fully start)
setTimeout(() => {
console.log('🚀 Running initial health checks...');
monitorProcessesSafe();
checkDomainHealthSafe();
checkNginxPortsSafe();
logActivity('STARTUP', 'Server Uptime', 'Monitoring agent started successfully with safe error recovery');
}, 5000);
app.listen(PORT, '0.0.0.0', () => {
console.log(`🟢 Server Uptime Agent running on port ${PORT}`);
console.log(`Dashboard: http://45.61.58.125:${PORT}`);
console.log(`⚡ Monitoring all processes every 15 seconds`);
console.log(`⚡ Auto-restart enabled for all stopped/errored processes`);
});