← back to Designer Wallcoverings
DW-Agents/dw-agents/agent-crash-loop-resolver/crash-loop-resolver.ts
595 lines
/**
* Crash Loop Resolver Agent
*
* Automatically detects and resolves crash-looping agents in the DW-Agents system.
*
* Features:
* - Monitors PM2 for agents with high restart counts
* - Analyzes error logs to identify root causes
* - Attempts automatic fixes for common issues
* - Tests fixes with Playwright
* - Reports status to master hub
*
* Port: 9895
*/
import express, { Request, Response } from 'express';
import { exec } from 'child_process';
import { promisify } from 'util';
import fs from 'fs/promises';
import path from 'path';
import { requireGlobalAuth } from '../shared-global-auth';
import cookieParser from 'cookie-parser';
import { chatMiddleware } from '../shared-chat-integration';
const execAsync = promisify(exec);
const app = express();
const PORT = 9895;
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(cookieParser());
app.use(chatMiddleware({
agentId: 'agent-crash-loop-resolver',
agentName: 'Crash Loop Resolver',
port: 9895,
category: 'agent'
}));
interface AgentStatus {
id: number;
name: string;
restarts: number;
status: string;
uptime: string;
pid: number;
}
interface CrashReport {
agent: string;
restarts: number;
logs: string;
errorPattern: string;
fixApplied: string;
success: boolean;
timestamp: string;
}
let crashReports: CrashReport[] = [];
let isResolving = false;
// Common crash patterns and their fixes
const CRASH_PATTERNS = [
{
pattern: /EADDRINUSE.*address already in use.*:(\d+)/,
name: 'Port Already In Use',
fix: async (agent: string, match: RegExpMatchArray) => {
const port = match[1];
// Kill process on that port
try {
await execAsync(`lsof -ti:${port} | xargs kill -9 2>/dev/null || true`);
return `Killed process on port ${port}`;
} catch (e) {
return `Failed to kill process on port ${port}`;
}
}
},
{
pattern: /Cannot find module ['"](.+)['"]/,
name: 'Missing Module',
fix: async (agent: string, match: RegExpMatchArray) => {
const module = match[1];
return `Missing module: ${module}. Needs npm install.`;
}
},
{
pattern: /AUTH_USERNAME is not defined|AUTH_PASSWORD is not defined/,
name: 'Undefined Auth Variables',
fix: async (agent: string) => {
const agentDir = agent.replace('dw-', '').replace(/^/, 'agent-');
const files = await execAsync(`find /root/Projects/Designer-Wallcoverings/DW-Agents/${agentDir} -name "*.ts" -type f`);
const fileList = files.stdout.trim().split('\n').filter(f => f);
for (const file of fileList) {
const content = await fs.readFile(file, 'utf-8');
if (content.includes('AUTH_USERNAME') || content.includes('AUTH_PASSWORD')) {
// Remove lines with AUTH_USERNAME/PASSWORD in console.log
const fixed = content.replace(/.*console\.log.*AUTH_(USERNAME|PASSWORD).*\n/g, '');
if (fixed !== content) {
await fs.writeFile(file, fixed);
return `Removed AUTH variable references from ${path.basename(file)}`;
}
}
}
return 'No AUTH variable issues found in code';
}
},
{
pattern: /Unexpected ["}]/,
name: 'Syntax Error - Orphaned Code',
fix: async (agent: string) => {
const agentDir = agent.replace('dw-', '').replace(/^/, 'agent-');
const files = await execAsync(`find /root/Projects/Designer-Wallcoverings/DW-Agents/${agentDir} -name "*.ts" -type f`);
const fileList = files.stdout.trim().split('\n').filter(f => f);
for (const file of fileList) {
const content = await fs.readFile(file, 'utf-8');
// Fix orphaned requireAuth function bodies
const fixed = content.replace(
/\/\/ REMOVED: const requireAuth.*?\{[^}]*if \(req\.session\.authenticated\).*?res\.redirect\(['"]\/login['"]\);[^}]*\}/gs,
'// REMOVED: const requireAuth = (req: any, res: any, next: any) => { ... };'
);
if (fixed !== content) {
await fs.writeFile(file, fixed);
return `Fixed orphaned code block in ${path.basename(file)}`;
}
}
return 'No orphaned code found';
}
},
{
pattern: /Port.*(\d+).*does not match.*(\d+)/i,
name: 'Port Mismatch',
fix: async (agent: string, match: RegExpMatchArray) => {
return `Port mismatch detected. Needs manual ecosystem.config.js alignment.`;
}
}
];
// Get all PM2 agents and their restart counts
async function getAgentStatus(): Promise<AgentStatus[]> {
try {
const { stdout } = await execAsync('pm2 jlist');
const agents = JSON.parse(stdout);
return agents
.filter((a: any) => a.name.startsWith('dw-'))
.map((a: any) => ({
id: a.pm_id,
name: a.name,
restarts: a.pm2_env.restart_time || 0,
status: a.pm2_env.status,
uptime: a.pm2_env.pm_uptime ? Math.floor((Date.now() - a.pm2_env.pm_uptime) / 1000) : 0,
pid: a.pid
}));
} catch (e) {
console.error('Failed to get agent status:', e);
return [];
}
}
// Analyze agent logs for crash patterns
async function analyzeLogs(agentName: string): Promise<{ logs: string; errorPattern: string }> {
const logFile = `/root/Projects/Designer-Wallcoverings/DW-Agents/logs/${agentName.replace('dw-', '')}-error.log`;
try {
const logs = await fs.readFile(logFile, 'utf-8');
const recentLogs = logs.split('\n').slice(-50).join('\n'); // Last 50 lines
// Check for known patterns
for (const pattern of CRASH_PATTERNS) {
if (pattern.pattern.test(recentLogs)) {
return { logs: recentLogs, errorPattern: pattern.name };
}
}
return { logs: recentLogs, errorPattern: 'Unknown' };
} catch (e) {
// Try PM2 logs
try {
const { stdout } = await execAsync(`pm2 logs ${agentName} --lines 50 --nostream --raw 2>&1`);
return { logs: stdout, errorPattern: 'Unknown' };
} catch (e2) {
return { logs: 'No logs available', errorPattern: 'No Logs' };
}
}
}
// Apply fix for detected pattern
async function applyFix(agentName: string, logs: string): Promise<string> {
for (const crashPattern of CRASH_PATTERNS) {
const match = logs.match(crashPattern.pattern);
if (match) {
console.log(`Detected pattern: ${crashPattern.name} for ${agentName}`);
const result = await crashPattern.fix(agentName, match);
return `${crashPattern.name}: ${result}`;
}
}
return 'No automatic fix available';
}
// Test agent with Playwright
async function testAgentWithPlaywright(agentName: string, port: number): Promise<boolean> {
try {
const testScript = `
const { chromium } = require('playwright');
(async () => {
const browser = await chromium.launch();
const page = await browser.newPage();
try {
const response = await page.goto('http://localhost:${port}', { timeout: 5000 });
await browser.close();
process.exit(response && response.ok() ? 0 : 1);
} catch (e) {
await browser.close();
process.exit(1);
}
})();
`;
await fs.writeFile('/tmp/test-agent.js', testScript);
await execAsync('node /tmp/test-agent.js', { timeout: 10000 });
return true;
} catch (e) {
return false;
}
}
// Get agent port from ecosystem config
async function getAgentPort(agentName: string): Promise<number | null> {
try {
const config = await fs.readFile('/root/Projects/Designer-Wallcoverings/DW-Agents/ecosystem.config.js', 'utf-8');
const match = config.match(new RegExp(`name:\\s*['"]${agentName}['"][^}]+PORT:\\s*(\\d+)`, 's'));
return match ? parseInt(match[1]) : null;
} catch (e) {
return null;
}
}
// Main crash loop resolver
async function resolveCrashLoops() {
if (isResolving) {
console.log('Already resolving crashes, skipping...');
return;
}
isResolving = true;
console.log('\n🔍 Scanning for crash-looping agents...');
try {
const agents = await getAgentStatus();
const crashLooping = agents.filter(a =>
a.restarts > 10 && a.uptime < 60 // More than 10 restarts and uptime < 60 seconds
);
if (crashLooping.length === 0) {
console.log('✅ No crash-looping agents detected');
return;
}
console.log(`⚠️ Found ${crashLooping.length} crash-looping agent(s):`);
crashLooping.forEach(a => console.log(` - ${a.name}: ${a.restarts} restarts`));
for (const agent of crashLooping) {
console.log(`\n🔧 Resolving ${agent.name}...`);
// 1. Analyze logs
const { logs, errorPattern } = await analyzeLogs(agent.name);
console.log(` Error pattern: ${errorPattern}`);
// 2. Apply fix
const fixResult = await applyFix(agent.name, logs);
console.log(` Fix: ${fixResult}`);
// 3. Restart agent
console.log(` Restarting ${agent.name}...`);
await execAsync(`pm2 restart ${agent.name}`);
// 4. Wait and check stability
await new Promise(resolve => setTimeout(resolve, 10000));
const newStatus = (await getAgentStatus()).find(a => a.name === agent.name);
let testPassed = false;
if (newStatus && newStatus.status === 'online' && newStatus.uptime > 5) {
// 5. Test with Playwright
const port = await getAgentPort(agent.name);
if (port) {
console.log(` Testing ${agent.name} on port ${port}...`);
testPassed = await testAgentWithPlaywright(agent.name, port);
}
}
const success = newStatus?.status === 'online' && newStatus.uptime > 5;
// Record report
const report: CrashReport = {
agent: agent.name,
restarts: agent.restarts,
logs: logs.slice(-500), // Last 500 chars
errorPattern,
fixApplied: fixResult,
success,
timestamp: new Date().toISOString()
};
crashReports.unshift(report);
if (crashReports.length > 20) crashReports = crashReports.slice(0, 20);
if (success) {
console.log(` ✅ ${agent.name} is now stable (uptime: ${newStatus.uptime}s, test: ${testPassed ? 'PASS' : 'N/A'})`);
// Send confirmation message
await sendConfirmationMessage(agent.name, testPassed);
} else {
console.log(` ❌ ${agent.name} still unstable`);
}
}
} catch (e) {
console.error('Error resolving crash loops:', e);
} finally {
isResolving = false;
}
}
// Send confirmation to master hub
async function sendConfirmationMessage(agentName: string, testPassed: boolean) {
try {
const message = `✅ Crash Loop Resolved: ${agentName}\n\n` +
`The agent has been fixed and is now running stably.\n` +
`Playwright Test: ${testPassed ? '✅ PASSED' : 'ℹ️ Not tested (no port found)'}\n` +
`Timestamp: ${new Date().toLocaleString()}`;
console.log('\n' + message);
// Could also POST to master hub or send notification here
// await fetch('http://localhost:9111/api/notifications', { ... });
} catch (e) {
console.error('Failed to send confirmation:', e);
}
}
// Dashboard HTML
app.get('/', requireGlobalAuth, (req, res) => {
const recentReports = crashReports.slice(0, 10);
res.send(`<!DOCTYPE html>
<html>
<head>
<title>Crash Loop Resolver</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
:root {
--primary: #e74c3c;
--primary-light: #c0392b;
--success: #27ae60;
--warning: #f39c12;
--bg: #ffffff;
--card-bg: #f8f9fa;
--text: #2c3e50;
}
body {
font-family: 'Segoe UI', system-ui, sans-serif;
background: var(--bg);
color: var(--text);
line-height: 1.6;
}
.header {
background: linear-gradient(135deg, var(--primary) 0%, var(--primary-light) 100%);
color: white;
padding: 25px 40px;
box-shadow: 0 4px 20px rgba(0,0,0,0.1);
}
.header h1 {
font-size: 2em;
display: flex;
align-items: center;
gap: 15px;
}
.container {
max-width: 1400px;
margin: 30px auto;
padding: 0 20px;
}
.controls {
background: var(--card-bg);
padding: 20px;
border-radius: 8px;
margin-bottom: 20px;
display: flex;
gap: 15px;
align-items: center;
}
.btn {
background: var(--primary);
color: white;
border: none;
padding: 12px 24px;
border-radius: 6px;
cursor: pointer;
font-size: 1em;
transition: all 0.3s;
}
.btn:hover {
background: var(--primary-light);
transform: translateY(-2px);
}
.btn.success {
background: var(--success);
}
.status-card {
background: var(--card-bg);
padding: 20px;
border-radius: 8px;
margin-bottom: 20px;
border-left: 4px solid var(--primary);
}
.reports-grid {
display: grid;
gap: 20px;
}
.report {
background: var(--card-bg);
padding: 20px;
border-radius: 8px;
border-left: 4px solid ${recentReports.length > 0 && recentReports[0].success ? 'var(--success)' : 'var(--primary)'};
}
.report.success {
border-left-color: var(--success);
}
.report.failed {
border-left-color: var(--primary);
}
.report-header {
display: flex;
justify-content: space-between;
margin-bottom: 15px;
font-weight: 600;
}
.report-details {
font-size: 0.9em;
color: #555;
margin-top: 10px;
}
.log-preview {
background: #000;
color: #0f0;
padding: 15px;
border-radius: 6px;
font-family: 'Courier New', monospace;
font-size: 0.85em;
margin-top: 10px;
max-height: 200px;
overflow-y: auto;
}
.badge {
display: inline-block;
padding: 4px 12px;
border-radius: 4px;
font-size: 0.85em;
font-weight: 600;
}
.badge.success {
background: var(--success);
color: white;
}
.badge.failed {
background: var(--primary);
color: white;
}
</style>
</head>
<body>
<div class="header">
<h1>🛠️ Crash Loop Resolver</h1>
<p>Automatic detection and resolution of crash-looping agents</p>
</div>
<div class="container">
<div class="controls">
<button class="btn" onclick="scanNow()">🔍 Scan Now</button>
<button class="btn success" onclick="location.reload()">🔄 Refresh</button>
<span style="margin-left: auto;">
${isResolving ? '⏳ Currently resolving...' : '✅ Ready'}
</span>
</div>
<div class="status-card">
<h2>📊 System Status</h2>
<p><strong>Total Reports:</strong> ${crashReports.length}</p>
<p><strong>Successful Fixes:</strong> ${crashReports.filter(r => r.success).length}</p>
<p><strong>Failed Attempts:</strong> ${crashReports.filter(r => !r.success).length}</p>
</div>
<h2 style="margin-bottom: 15px;">Recent Resolution Attempts</h2>
${recentReports.length === 0 ?
'<div class="status-card"><p>No crash loop resolutions yet. System is stable! 🎉</p></div>' :
'<div class="reports-grid">' + recentReports.map(r => `
<div class="report ${r.success ? 'success' : 'failed'}">
<div class="report-header">
<span><strong>${r.agent}</strong></span>
<span class="badge ${r.success ? 'success' : 'failed'}">
${r.success ? '✅ RESOLVED' : '❌ FAILED'}
</span>
</div>
<div class="report-details">
<p><strong>Restarts:</strong> ${r.restarts}</p>
<p><strong>Error Pattern:</strong> ${r.errorPattern}</p>
<p><strong>Fix Applied:</strong> ${r.fixApplied}</p>
<p><strong>Timestamp:</strong> ${new Date(r.timestamp).toLocaleString()}</p>
</div>
${r.logs ? `<div class="log-preview">${r.logs.replace(/</g, '<').replace(/>/g, '>')}</div>` : ''}
</div>
`).join('') + '</div>'
}
</div>
<script>
async function scanNow() {
const btn = event.target;
btn.disabled = true;
btn.textContent = '⏳ Scanning...';
try {
await fetch('/api/resolve', { method: 'POST' });
setTimeout(() => location.reload(), 2000);
} catch (e) {
alert('Failed to trigger scan: ' + e.message);
btn.disabled = false;
btn.textContent = '🔍 Scan Now';
}
}
</script>
</body>
</html>`);
});
// API: Trigger manual resolution
app.post('/api/resolve', requireGlobalAuth, async (req, res) => {
if (isResolving) {
return res.json({ success: false, message: 'Already resolving' });
}
// Run in background
resolveCrashLoops().catch(console.error);
res.json({ success: true, message: 'Scan started' });
});
// API: Get current status
app.get('/api/status', requireGlobalAuth, async (req, res) => {
const agents = await getAgentStatus();
const crashLooping = agents.filter(a => a.restarts > 10 && a.uptime < 60);
res.json({
totalAgents: agents.length,
crashLooping: crashLooping.length,
agents: crashLooping,
recentReports: crashReports.slice(0, 5),
isResolving
});
});
// Metrics endpoint for daily reporting
app.get("/api/metrics", requireGlobalAuth, (req: Request, res: Response) => {
res.json({
status: "online",
uptime: process.uptime(),
responseTime: 0,
tasksCompleted: crashReports.filter(r => r.success).length,
errorsToday: crashReports.filter(r => !r.success).length,
lastActivity: crashReports[0]?.timestamp || new Date().toISOString(),
qnaReadiness: 100
});
});
// Auto-scan every 5 minutes
setInterval(() => {
resolveCrashLoops().catch(console.error);
}, 5 * 60 * 1000);
app.listen(PORT, '0.0.0.0', () => {
console.log('');
console.log('🛠️ Crash Loop Resolver Agent');
console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━');
console.log('🌍 External: http://45.61.58.125:' + PORT);
console.log('🏠 Local: http://localhost:' + PORT);
console.log('');
console.log('✅ Monitoring for crash loops every 5 minutes...');
console.log('');
});