← back to Timeout Agent
server.js
1203 lines
/**
* Timeout Agent — 24/7 PM2 Process Watchdog
* Port: 9614 | Auth: admin/DWSecure2024!
*
* Monitors ALL PM2 processes every 60 seconds.
* Auto-restarts: errored, stopped, stuck, unresponsive, and crash-looping processes.
* Logs every action to PostgreSQL. Dashboard with live status.
*/
const express = require('express');
const helmet = require('helmet');
const { Pool } = require('pg');
const { execSync } = require('child_process');
const http = require('http');
const https = require('https');
const app = express();
// Security headers via helmet (added 2026-05-04 overnight YOLO loop)
app.use(helmet({ contentSecurityPolicy: false }));
const PORT = process.env.PORT || 9614;
const AGENT_COLOR = '#F59E0B'; // Amber/Warning yellow
const AGENT_NAME = 'Timeout Agent';
const pool = new Pool({ connectionString: (process.env.DATABASE_URL || 'postgresql://dw_admin@127.0.0.1:15432/dw_unified') });
const SLACK_WEBHOOK = 'https://hooks.slack.com/services/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';
// Config
const CHECK_INTERVAL = 60 * 1000; // 60 seconds
const MAX_RESTARTS = 20; // Threshold for crash loop
const HEALTH_TIMEOUT = 8000; // 8s health check timeout
const COOLDOWN_MS = 5 * 60 * 1000; // 5 min cooldown between restarts of same process
const MAX_RESTARTS_PER_HOUR = 3; // Don't restart same process more than 3x/hour
// Known health endpoints by PM2 name patterns
const HEALTH_PORTS = {};
// State
const watchState = {
enabled: true,
running: false,
lastCheck: null,
totalChecks: 0,
totalRestarts: 0,
recentActions: [], // last 50 actions
processStatus: {}, // pm2_name -> { status, restarts, lastRestart, lastHealthy, issues }
cooldowns: {} // pm2_name -> last restart timestamp
};
// ── Auth ──
function basicAuth(req, res, next) {
if (req.path === '/api/status') return next();
const auth = req.headers.authorization;
if (!auth || !auth.startsWith('Basic ')) {
res.setHeader('WWW-Authenticate', 'Basic realm="Timeout Agent"');
return res.status(401).send('Authentication required');
}
const [u, p] = Buffer.from(auth.split(' ')[1], 'base64').toString().split(':');
if (u === 'admin' && p === 'DWSecure2024!') return next();
res.setHeader('WWW-Authenticate', 'Basic realm="Timeout Agent"');
return res.status(401).send('Invalid credentials');
}
app.use(basicAuth);
app.use(express.json());
// ── SSE live logs ──
const sseClients = [];
function broadcast(msg) {
const data = JSON.stringify({ ts: new Date().toISOString(), msg });
sseClients.forEach(res => res.write(`data: ${data}\n\n`));
console.log(`[Timeout] ${msg}`);
}
app.get('/api/events', (req, res) => {
res.setHeader('Content-Type', 'text/event-stream');
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.flushHeaders();
sseClients.push(res);
req.on('close', () => { const i = sseClients.indexOf(res); if (i >= 0) sseClients.splice(i, 1); });
});
// ── DB init ──
async function initDB() {
await pool.query(`
CREATE TABLE IF NOT EXISTS timeout_agent_log (
id SERIAL PRIMARY KEY,
pm2_name TEXT NOT NULL,
pm2_id INTEGER,
action TEXT NOT NULL,
reason TEXT,
prev_status TEXT,
prev_restarts INTEGER,
success BOOLEAN DEFAULT true,
error_msg TEXT,
created_at TIMESTAMP DEFAULT NOW()
)
`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_timeout_log_name ON timeout_agent_log(pm2_name)`);
await pool.query(`CREATE INDEX IF NOT EXISTS idx_timeout_log_created ON timeout_agent_log(created_at)`);
// Add diagnosis column if not exists
try {
await pool.query(`ALTER TABLE timeout_agent_log ADD COLUMN IF NOT EXISTS diagnosis TEXT`);
} catch (e) {}
broadcast('Database tables ready');
}
// ── Slack notify — DISABLED 2026-03-25 per Steve ──
function slackNotify(text) {
// Slack notifications disabled — too noisy with auto-restart chatter
// To re-enable: uncomment the block below
/*
try {
const url = new URL(SLACK_WEBHOOK);
const body = JSON.stringify({ text });
const opts = {
hostname: url.hostname, path: url.pathname, method: 'POST',
headers: { 'Content-Type': 'application/json' }
};
const req = https.request(opts, () => {});
req.on('error', () => {});
req.write(body);
req.end();
} catch (e) {}
*/
}
// ── Get PM2 process list ──
function getPM2List() {
try {
const raw = execSync('pm2 jlist 2>/dev/null', { timeout: 15000, maxBuffer: 5 * 1024 * 1024 }).toString();
return JSON.parse(raw);
} catch (e) {
broadcast('ERROR: Failed to get PM2 list: ' + e.message);
return [];
}
}
// ── Read PM2 error logs for a process ──
function readProcessLogs(name, lines = 50) {
try {
// Try error log first (most useful for diagnosing crashes)
const errLog = execSync(`pm2 logs ${name} --err --lines ${lines} --nostream 2>/dev/null`, {
timeout: 10000, maxBuffer: 512 * 1024
}).toString().trim();
// Also grab stdout for context
let outLog = '';
try {
outLog = execSync(`pm2 logs ${name} --out --lines 20 --nostream 2>/dev/null`, {
timeout: 10000, maxBuffer: 256 * 1024
}).toString().trim();
} catch (e) {}
return { errLog, outLog, combined: (errLog + '\n' + outLog).trim() };
} catch (e) {
return { errLog: '', outLog: '', combined: '' };
}
}
// ── Analyze logs and diagnose the issue ──
function diagnoseFromLogs(name, logs) {
const text = logs.combined.toLowerCase();
const diagnosis = { issue: 'unknown', fix: null, details: '' };
// Port already in use
const portMatch = logs.combined.match(/EADDRINUSE.*?:(\d+)/i) || logs.combined.match(/address already in use.*?:(\d+)/i);
if (portMatch) {
diagnosis.issue = 'port_in_use';
diagnosis.port = portMatch[1];
diagnosis.fix = 'kill_port';
diagnosis.details = `Port ${portMatch[1]} already in use`;
return diagnosis;
}
// Module not found — needs npm install
if (text.includes('cannot find module') || text.includes('module_not_found') || text.includes('cannot find package')) {
const modMatch = logs.combined.match(/Cannot find (?:module|package) '([^']+)'/i);
diagnosis.issue = 'missing_module';
diagnosis.module = modMatch ? modMatch[1] : 'unknown';
diagnosis.fix = 'npm_install';
diagnosis.details = `Missing module: ${diagnosis.module}`;
return diagnosis;
}
// Database connection refused
if (text.includes('econnrefused') && (text.includes('5432') || text.includes('postgres'))) {
diagnosis.issue = 'db_connection';
diagnosis.fix = 'check_postgres';
diagnosis.details = 'PostgreSQL connection refused';
return diagnosis;
}
// Syntax error or reference error (code bug — can't auto-fix)
if (text.includes('syntaxerror:') || text.includes('referenceerror:')) {
const errMatch = logs.combined.match(/(SyntaxError|ReferenceError):\s*(.+)/i);
diagnosis.issue = 'code_error';
diagnosis.fix = null;
diagnosis.details = errMatch ? `${errMatch[1]}: ${errMatch[2].substring(0, 100)}` : 'Code error detected';
return diagnosis;
}
// Out of memory
if (text.includes('javascript heap out of memory') || text.includes('allocation failed')) {
diagnosis.issue = 'out_of_memory';
diagnosis.fix = 'increase_memory';
diagnosis.details = 'JavaScript heap out of memory';
return diagnosis;
}
// Permission denied
if (text.includes('eacces') || text.includes('permission denied')) {
diagnosis.issue = 'permission_denied';
diagnosis.fix = null;
diagnosis.details = 'Permission denied error';
return diagnosis;
}
// TypeScript / tsx issues
if (text.includes('tsx') && (text.includes('err_unknown_file_extension') || text.includes('.ts'))) {
diagnosis.issue = 'tsx_issue';
diagnosis.fix = 'npm_install';
diagnosis.details = 'TypeScript execution issue (tsx)';
return diagnosis;
}
// Generic crash — just try restart
if (text.includes('error') || text.includes('fatal') || text.includes('throw')) {
const errLine = logs.combined.split('\n').find(l => /error|fatal|throw/i.test(l)) || '';
diagnosis.issue = 'generic_error';
diagnosis.fix = 'restart_only';
diagnosis.details = errLine.substring(0, 150) || 'Generic error detected';
return diagnosis;
}
diagnosis.fix = 'restart_only';
diagnosis.details = 'No specific error pattern found, will attempt restart';
return diagnosis;
}
// ── Apply fix based on diagnosis ──
function applyFix(name, diagnosis) {
broadcast(` 🔧 Applying fix for ${name}: ${diagnosis.issue} → ${diagnosis.fix || 'none'}`);
try {
switch (diagnosis.fix) {
case 'kill_port': {
const port = diagnosis.port;
broadcast(` 🔧 Checking and resolving port ${port} conflict...`);
try {
// Try multiple methods to find process on port
let pids = '';
try { pids = execSync(`lsof -ti:${port} 2>/dev/null`, { timeout: 5000 }).toString().trim(); } catch (e) {}
if (!pids) {
try { pids = execSync(`fuser ${port}/tcp 2>/dev/null`, { timeout: 5000 }).toString().trim(); } catch (e) {}
}
if (!pids) {
try {
const ss = execSync(`ss -tlnp sport = :${port} 2>/dev/null`, { timeout: 5000 }).toString();
const pidMatch = ss.match(/pid=(\d+)/);
if (pidMatch) pids = pidMatch[1];
} catch (e) {}
}
if (pids) {
const pidList = pids.split(/[\s\n]+/).filter(p => p && /^\d+$/.test(p));
// Safety check: if the PID belongs to another healthy PM2 process, skip the kill.
// Killing a healthy PM2 agent to fix a port conflict is worse than the crash loop.
let pm2Procs = [];
try { pm2Procs = getPM2List(); } catch (ignore) {}
const pm2PidSet = new Set(
pm2Procs.filter(p => p.name !== name && p.pid > 0).map(p => String(p.pid))
);
const pm2Owners = pidList.filter(pid => pm2PidSet.has(pid));
if (pm2Owners.length > 0) {
const ownerProc = pm2Procs.find(p => pm2Owners.some(pid => String(p.pid) === pid));
const ownerName = ownerProc ? ownerProc.name : 'another PM2 process';
broadcast(` 🔧 SKIPPED: port ${port} is held by PM2 process "${ownerName}" — port conflict in ${name} must be fixed in code`);
return { applied: false, action: `port conflict: port ${port} owned by "${ownerName}" — requires code fix in ${name}` };
}
// Safe to kill — PIDs are not PM2-managed
for (const pid of pidList) {
try {
execSync(`kill -9 ${pid} 2>/dev/null`, { timeout: 5000 });
broadcast(` 🔧 Killed external PID ${pid} on port ${port}`);
} catch (e) {}
}
execSync('sleep 2');
} else {
// No external process found - kill all PM2 processes with this name first
broadcast(` 🔧 No external process on port ${port}, will clean up via pm2 delete`);
try { execSync(`pm2 delete ${name} 2>/dev/null`, { timeout: 10000 }); } catch (e) {}
execSync('sleep 2');
}
} catch (e) {
broadcast(` 🔧 Port cleanup error: ${e.message.substring(0, 80)}`);
}
return { applied: true, action: `cleaned port ${port}` };
}
case 'npm_install': {
const cwd = findProjectDir(name);
if (cwd) {
broadcast(` 🔧 Running npm install in ${cwd}...`);
try {
execSync(`cd "${cwd}" && npm install --no-audit --no-fund 2>&1 | tail -5`, {
timeout: 60000, maxBuffer: 1024 * 1024
});
broadcast(` 🔧 npm install completed in ${cwd}`);
return { applied: true, action: `npm install in ${cwd}` };
} catch (e) {
broadcast(` 🔧 npm install failed: ${e.message.substring(0, 100)}`);
return { applied: false, action: 'npm install failed' };
}
}
return { applied: false, action: 'could not find project directory' };
}
case 'check_postgres': {
broadcast(` 🔧 Checking PostgreSQL...`);
try {
execSync('pg_isready -h 127.0.0.1 -p 5432 2>/dev/null', { timeout: 5000 });
broadcast(` 🔧 PostgreSQL is running — connection issue may be transient`);
} catch (e) {
broadcast(` 🔧 PostgreSQL appears down, attempting restart...`);
try {
execSync('systemctl restart postgresql 2>/dev/null || service postgresql restart 2>/dev/null', { timeout: 15000 });
execSync('sleep 3');
broadcast(` 🔧 PostgreSQL restarted`);
} catch (e2) {
broadcast(` 🔧 PostgreSQL restart failed`);
}
}
return { applied: true, action: 'checked/restarted postgres' };
}
case 'increase_memory': {
broadcast(` 🔧 Will attempt restart with increased memory limit`);
return { applied: true, action: 'will increase memory on restart' };
}
case 'restart_only':
return { applied: true, action: 'no fix needed, restart only' };
default:
return { applied: false, action: `no automated fix for ${diagnosis.issue}` };
}
} catch (e) {
broadcast(` 🔧 Fix failed: ${e.message.substring(0, 100)}`);
return { applied: false, action: e.message.substring(0, 100) };
}
}
// ── Find project directory for a PM2 process ──
function findProjectDir(name) {
const fs = require('fs');
const dirs = [
`/root/DW-Agents/${name}`,
`/root/Projects/${name}`,
`/root/Projects/Designer-Wallcoverings/DW-Programming/${name}`,
];
for (const d of dirs) {
try {
fs.accessSync(d + '/package.json');
return d;
} catch (e) {}
}
// Try to get it from PM2 process info
try {
const raw = execSync(`pm2 show ${name} 2>/dev/null`, { timeout: 10000 }).toString();
const cwdMatch = raw.match(/exec cwd\s*│\s*(.+)/);
if (cwdMatch) {
const cwd = cwdMatch[1].trim();
try { fs.accessSync(cwd); return cwd; } catch (e) {}
}
} catch (e) {}
return null;
}
// ── Verify process is actually running after restart ──
function verifyProcessRunning(name, waitMs = 5000) {
// Wait for process to stabilize
execSync(`sleep ${Math.ceil(waitMs / 1000)}`);
const procs = getPM2List();
const proc = procs.find(p => p.name === name);
if (!proc) return { running: false, status: 'not_found' };
const status = proc.pm2_env?.status || 'unknown';
const uptime = Date.now() - (proc.pm2_env?.pm_uptime || 0);
return {
running: status === 'online',
status,
uptimeMs: uptime,
restarts: proc.pm2_env?.restart_time || 0
};
}
// ── SMART RESTART: Diagnose → Fix → Restart → Verify → Retry ──
async function smartRestart(name, id, reason, prevStatus, prevRestarts) {
const MAX_ATTEMPTS = 2;
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
broadcast(`🔍 [${name}] Attempt ${attempt}/${MAX_ATTEMPTS} — Reading logs...`);
// Step 1: Read logs
const logs = readProcessLogs(name);
if (logs.combined) {
const logPreview = logs.errLog.split('\n').slice(-5).join(' | ').substring(0, 200);
broadcast(`🔍 [${name}] Error log: ${logPreview || '(empty)'}`);
} else {
broadcast(`🔍 [${name}] No logs found`);
}
// Step 2: Diagnose
const diagnosis = diagnoseFromLogs(name, logs);
broadcast(`🔍 [${name}] Diagnosis: ${diagnosis.issue} — ${diagnosis.details}`);
// Step 3: Apply fix if available
if (diagnosis.fix && diagnosis.fix !== 'restart_only') {
const fixResult = applyFix(name, diagnosis);
broadcast(`🔧 [${name}] Fix result: ${fixResult.applied ? '✓' : '✗'} ${fixResult.action}`);
await logAction(name, id, 'auto_fix', `${diagnosis.issue}: ${fixResult.action}`, prevStatus, prevRestarts, fixResult.applied, fixResult.applied ? null : fixResult.action);
}
// Step 4: Restart
broadcast(`🔄 [${name}] Restarting (attempt ${attempt})...`);
let result;
// Use hard restart on second attempt or crash loops
if (attempt === 2 || prevRestarts > MAX_RESTARTS) {
result = hardRestart(name, id);
await logAction(name, id, 'hard_restart', `attempt ${attempt}: ${diagnosis.issue} — ${diagnosis.details.substring(0, 100)}`, prevStatus, prevRestarts, result.success, result.error || null);
} else {
result = restartProcess(name, id);
await logAction(name, id, 'restart', `attempt ${attempt}: ${diagnosis.issue} — ${diagnosis.details.substring(0, 100)}`, prevStatus, prevRestarts, result.success, result.error || null);
}
if (!result.success) {
broadcast(`❌ [${name}] Restart command failed: ${result.error?.substring(0, 100)}`);
continue;
}
// Step 5: Verify it's actually running
broadcast(`⏳ [${name}] Verifying restart (waiting 5s)...`);
const verify = verifyProcessRunning(name, 5000);
if (verify.running) {
broadcast(`✅ [${name}] Successfully restarted! Status: ${verify.status}, uptime: ${Math.round(verify.uptimeMs/1000)}s`);
slackNotify(`:white_check_mark: *Timeout Agent* fixed & restarted \`${name}\` — was ${prevStatus} (${diagnosis.issue}: ${diagnosis.details.substring(0, 80)})`);
watchState.cooldowns[name] = Date.now();
watchState.totalRestarts++;
return { success: true, attempts: attempt, diagnosis };
}
broadcast(`⚠️ [${name}] Still ${verify.status} after restart attempt ${attempt}`);
// On first failure, check logs again for NEW errors before retry
if (attempt < MAX_ATTEMPTS) {
broadcast(`🔍 [${name}] Checking post-restart logs before retry...`);
const newLogs = readProcessLogs(name, 20);
const newDiag = diagnoseFromLogs(name, newLogs);
if (newDiag.issue !== diagnosis.issue) {
broadcast(`🔍 [${name}] New issue detected: ${newDiag.issue} — ${newDiag.details}`);
// Apply new fix
if (newDiag.fix && newDiag.fix !== 'restart_only') {
applyFix(name, newDiag);
}
}
}
}
// All attempts failed
broadcast(`❌ [${name}] Failed after ${MAX_ATTEMPTS} attempts`);
slackNotify(`:x: *Timeout Agent* could not restart \`${name}\` after ${MAX_ATTEMPTS} attempts — ${reason}`);
watchState.cooldowns[name] = Date.now();
return { success: false, attempts: MAX_ATTEMPTS };
}
// ── Simple restart (for non-smart cases) ──
function restartProcess(name, id) {
try {
// Try name first, fall back to id
try {
execSync(`pm2 restart ${name} 2>/dev/null`, { timeout: 30000 });
} catch (e) {
execSync(`pm2 restart ${id} 2>/dev/null`, { timeout: 30000 });
}
return { success: true };
} catch (e) {
return { success: false, error: e.message };
}
}
// ── Delete and re-start for crash loops ──
function hardRestart(name, id) {
try {
// Step 1: Delete by name (or id fallback)
try { execSync(`pm2 delete ${name} 2>/dev/null`, { timeout: 15000 }); } catch (e) {
try { execSync(`pm2 delete ${id} 2>/dev/null`, { timeout: 15000 }); } catch (e2) {}
}
// Step 2: Start fresh from ecosystem config or project dir
const ecoPath = findEcosystem(name);
if (ecoPath) {
execSync(`pm2 start ${ecoPath} --only ${name} 2>&1`, { timeout: 30000 });
return { success: true };
}
// Step 3: No ecosystem config — find project dir and start script directly
const projDir = findProjectDir(name);
if (projDir) {
const fs = require('fs');
const pkg = JSON.parse(fs.readFileSync(`${projDir}/package.json`, 'utf8'));
const script = pkg.main || pkg.scripts?.start?.replace('node ', '') || 'server.js';
execSync(`pm2 start ${projDir}/${script} --name ${name} 2>&1`, { timeout: 30000 });
return { success: true };
}
// Step 4: Last resort — just try pm2 start by name
execSync(`pm2 start ${name} 2>&1`, { timeout: 30000 });
return { success: true };
} catch (e) {
return { success: false, error: e.message.substring(0, 200) };
}
}
// ── Find ecosystem.config for a process ──
function findEcosystem(name) {
const fs = require('fs');
const paths = [
`/root/DW-Agents/${name}/ecosystem.config.cjs`,
`/root/DW-Agents/${name}/ecosystem.config.js`,
];
for (const p of paths) {
try { fs.accessSync(p); return p; } catch (e) {}
}
return null;
}
// ── Check if process was recently restarted (cooldown) ──
function isOnCooldown(name) {
const last = watchState.cooldowns[name];
if (!last) return false;
return (Date.now() - last) < COOLDOWN_MS;
}
// ── Count restarts for a process in the last hour ──
async function restartsLastHour(name) {
try {
const { rows } = await pool.query(
"SELECT COUNT(*) as cnt FROM timeout_agent_log WHERE pm2_name=$1 AND action='restart' AND created_at > NOW() - INTERVAL '1 hour'",
[name]
);
return parseInt(rows[0].cnt) || 0;
} catch (e) {
return 0;
}
}
// ── Log action to DB ──
async function logAction(pm2Name, pm2Id, action, reason, prevStatus, prevRestarts, success, errorMsg) {
try {
await pool.query(
'INSERT INTO timeout_agent_log (pm2_name, pm2_id, action, reason, prev_status, prev_restarts, success, error_msg) VALUES ($1,$2,$3,$4,$5,$6,$7,$8)',
[pm2Name, pm2Id, action, reason, prevStatus, prevRestarts, success, errorMsg]
);
} catch (e) {}
// Keep in-memory recent actions
watchState.recentActions.unshift({
ts: new Date().toISOString(), name: pm2Name, action, reason, success
});
if (watchState.recentActions.length > 50) watchState.recentActions.length = 50;
}
// ── ALLOWLIST: When non-empty, ONLY these processes are monitored ──
// Set to [] to monitor everything (default). Supports exact names and regex patterns.
const ALLOWED_PROCESSES_FILTER = [];
function isAllowed(name) {
if (ALLOWED_PROCESSES_FILTER.length === 0) return true;
return ALLOWED_PROCESSES_FILTER.some(p => p instanceof RegExp ? p.test(name) : name === p);
}
// ── Processes to skip (don't try to restart these) ──
const SKIP_PROCESSES = new Set([
'timeout-agent', // Don't restart ourselves
'bubbe', // Retired
'bubbe-lang-service', // Retired
'boardroom-3d', // Manual control only — eats compute
'boardroom-agent', // Manual control only — eats compute
'agent-cronjob', // Manual control only — eats compute
'schedule-engine', // Manual control only — eats compute
// ── Non-essential: stopped to free RAM (start manually when needed) ──
'room-setting-app', // 411MB — start for room renders only
'room-setting-agent', // 94MB — start for room renders only
'furniture-scene-agent', // 90MB — start for furniture renders only
'annie-arbitrage', // 103MB — start for price scanning only
'ringcentral-agent', // 84MB — phone integration
'instagram-agent', // 78MB — social media
'tasha-agent', // 77MB — forecasting
'zendesk-agent', // 63MB — support tickets
'slack-dm-viewer', // 62MB — Slack viewer
// ── Vendor catalog agents (start for scraping jobs) ──
'brewster-york-catalog', 'rl-catalog-9608', 'thibaut-agent',
'knox-agent', 'cleo-agent', 'sasha-agent', 'gb-agent',
'maya-agent', 'artie-agent', 'pj-agent', 'elise-agent',
'beau-agent', 'ines-agent', 'kira-agent', 'graham-brown-catalog',
'york-contract-catalog', 'william-watch-agent',
'momentum-crawler', 'momentum-pricer', // Long-running scrape jobs
]);
// Processes that are known legacy errored and shouldn't be force-restarted
// (they fail due to cluster mode + tsx incompatibility)
const LEGACY_ERRORED = new Set([
'dw-accounting', 'dw-agent-sku', 'dw-bigquery-gov', 'dw-crash-loop-resolver',
'dw-digital-samples', 'dw-log-monitor', 'dw-marketing',
'dw-needs-attention', 'dw-purchasing-office', 'dw-shopify-store',
'dw-skills-manager', 'dw-trend-research', 'dw-ui-manager', 'dw-zendesk-chat',
'dw-legal-team'
// REBUILT v2.0: dw-master-hub, dw-dashboard-monitor, dw-agent-directory now active
]);
// Production websites — NEVER hard-restart these (Cloudflare-mapped, always accumulate restarts)
const PRODUCTION_SITES = new Set([
'flockedwallpaper', 'glassbeadedwallpaper',
'grassclothwallcoverings-8080', 'grassclothwallcoverings-9080',
'grassclothwallpaper', 'novasuede-website', 'bubbe', 'bubbe-lang-service'
]);
// Batch jobs — one-shot scripts that exit when done. Stopped = FINISHED, not broken.
// The Warden must NEVER restart these. They run via cron or manually, not as persistent services.
const BATCH_JOBS = new Set([
'ai-tagger-all', 'Tag Fixer', 'Global to Custom Migrator',
'body-html-scan', 'body-html-fix', 'body-html-audit',
'am-recrawl', 'push-vendor-images', 'pl-image-sanitizer',
'hollywood-style-tagger', 'gemini-catalog-tagger', 'gemini-tag-engine',
'global-to-custom-v2', 'harlequin-backfill', 'gap-agent',
'gemini-classifier'
]);
// Track restart counts to detect ACTIVE crash looping (count increasing rapidly)
const prevRestartCounts = {};
// ── Main watchdog loop ──
async function checkProcesses() {
if (watchState.running) return;
watchState.running = true;
watchState.totalChecks++;
watchState.lastCheck = new Date().toISOString();
try {
const procs = getPM2List();
if (!procs.length) {
watchState.running = false;
return;
}
let restarted = 0;
let checked = 0;
let issues = [];
for (const proc of procs) {
const name = proc.name;
const id = proc.pm_id;
const status = proc.pm2_env?.status || 'unknown';
const restarts = proc.pm2_env?.restart_time || 0;
const uptime = proc.pm2_env?.pm_uptime || 0;
const memory = proc.monit?.memory || 0;
const memMB = Math.round(memory / 1024 / 1024);
checked++;
// Skip ourselves and known legacy
if (SKIP_PROCESSES.has(name)) continue;
// Skip if not in allowlist (when filter is active)
if (!isAllowed(name)) continue;
// Track process status
if (!watchState.processStatus[name]) {
watchState.processStatus[name] = { status, restarts, lastRestart: null, lastHealthy: null, issues: [] };
}
const ps = watchState.processStatus[name];
ps.status = status;
ps.restarts = restarts;
ps.memMB = memMB;
ps.issues = [];
// Check 1: Errored processes → SMART RESTART
if (status === 'errored') {
if (LEGACY_ERRORED.has(name)) {
ps.issues.push('legacy-errored');
continue;
}
if (isOnCooldown(name)) {
ps.issues.push('errored-cooldown');
continue;
}
const hourCount = await restartsLastHour(name);
if (hourCount >= MAX_RESTARTS_PER_HOUR) {
ps.issues.push('errored-rate-limited');
continue;
}
broadcast(`🔍 SMART RESTART: ${name} (id:${id}) — errored state`);
const result = await smartRestart(name, id, 'errored state', status, restarts);
restarted++;
if (result.success) {
ps.lastRestart = new Date().toISOString();
ps.lastDiagnosis = result.diagnosis;
}
continue;
}
// Check 2: Stopped processes → SMART RESTART (but skip batch jobs!)
if (status === 'stopped') {
if (LEGACY_ERRORED.has(name)) continue;
if (BATCH_JOBS.has(name)) continue; // Batch jobs stop when done — that's correct behavior
if (isOnCooldown(name)) continue;
const hourCount = await restartsLastHour(name);
if (hourCount >= MAX_RESTARTS_PER_HOUR) continue;
broadcast(`🔍 SMART RESTART: ${name} (id:${id}) — stopped`);
const result = await smartRestart(name, id, 'stopped', status, restarts);
restarted++;
if (result.success) {
ps.lastRestart = new Date().toISOString();
ps.lastDiagnosis = result.diagnosis;
}
continue;
}
// Check 3: Active crash loop detection → STOP (not restart!)
// A process restarting 20+ times is broken. Restarting it again just feeds the loop.
// Instead: STOP the process and alert Steve so a human can investigate.
if (status === 'online' && restarts > MAX_RESTARTS) {
const prevCount = prevRestartCounts[name] || 0;
const delta = restarts - prevCount;
prevRestartCounts[name] = restarts;
if (PRODUCTION_SITES.has(name)) continue;
if (LEGACY_ERRORED.has(name)) continue;
if (delta < 5) continue;
ps.issues.push('crash-loop-stopped');
// STOP the process — do NOT restart it
try {
execSync(`pm2 stop "${name.replace(/"/g, '')}" 2>/dev/null`, { timeout: 15000 });
broadcast(`CRASH LOOP STOPPED: ${name} (id:${id}) — ${restarts.toLocaleString()} restarts, +${delta} in last check. Process halted.`);
await logAction(name, id, 'crash_loop_stop', `Halted: ${restarts} restarts, delta +${delta}`, status, restarts, true, null);
// Alert Steve via Slack
try {
const alertMsg = `CRASH LOOP HALTED: ${name} (${restarts.toLocaleString()} restarts, +${delta}/min). Process stopped by Warden. Run pm2 start ${name} to restart after fixing the root cause.`;
const url = new URL(SLACK_WEBHOOK);
const body = JSON.stringify({ text: alertMsg });
const req = (url.protocol === 'https:' ? https : http).request({ hostname: url.hostname, port: url.port || 443, path: url.pathname, method: 'POST', headers: { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) } });
req.write(body);
req.end();
} catch (slackErr) { /* non-critical */ }
} catch (stopErr) {
broadcast(`CRASH LOOP: Failed to stop ${name}: ${stopErr.message}`);
await logAction(name, id, 'crash_loop_stop', `Failed to halt: ${stopErr.message}`, status, restarts, false, stopErr.message);
}
continue;
}
// Track restart counts for delta detection
prevRestartCounts[name] = restarts;
// Check 4: Online but possibly stuck (very high memory) → SMART RESTART
if (status === 'online' && memMB > 450 && !PRODUCTION_SITES.has(name)) {
ps.issues.push('high-memory-' + memMB + 'MB');
if (isOnCooldown(name)) continue;
const hourCount = await restartsLastHour(name);
if (hourCount >= MAX_RESTARTS_PER_HOUR) continue;
broadcast(`🔍 SMART RESTART: ${name} (id:${id}) — high memory ${memMB}MB`);
const result = await smartRestart(name, id, `high memory: ${memMB}MB`, status, restarts);
restarted++;
if (result.success) {
ps.lastRestart = new Date().toISOString();
ps.lastDiagnosis = result.diagnosis;
}
continue;
}
// If online and healthy, mark it
if (status === 'online') {
ps.lastHealthy = new Date().toISOString();
}
}
if (restarted > 0) {
broadcast(`Check complete: ${checked} processes, ${restarted} restarted`);
// Save PM2 state after restarts
try { execSync('pm2 save 2>/dev/null', { timeout: 10000 }); } catch (e) {}
}
} catch (err) {
broadcast('ERROR in watchdog loop: ' + err.message);
} finally {
watchState.running = false;
}
}
// ── API Endpoints ──
app.get('/api/status', (req, res) => {
const online = Object.values(watchState.processStatus).filter(p => p.status === 'online').length;
const errored = Object.values(watchState.processStatus).filter(p => p.status === 'errored').length;
const stopped = Object.values(watchState.processStatus).filter(p => p.status === 'stopped').length;
res.json({
agent: AGENT_NAME, port: PORT, status: 'online',
watching: watchState.enabled,
lastCheck: watchState.lastCheck,
totalChecks: watchState.totalChecks,
totalRestarts: watchState.totalRestarts,
processes: { online, errored, stopped, total: Object.keys(watchState.processStatus).length },
timestamp: new Date().toISOString()
});
});
app.get('/api/processes', (req, res) => {
const list = Object.entries(watchState.processStatus).map(([name, ps]) => ({
name, ...ps
}));
const sort = String(req.query.sort || 'status').toLowerCase();
const statusRank = (s) => s === 'errored' ? 0 : s === 'stopped' ? 1 : s === 'online' ? 2 : 3;
const byName = (a, b) => a.name.localeCompare(b.name);
switch (sort) {
case 'name':
case 'name-asc':
list.sort(byName);
break;
case 'name-desc':
list.sort((a, b) => b.name.localeCompare(a.name));
break;
case 'restarts-desc':
list.sort((a, b) => (b.restarts || 0) - (a.restarts || 0) || byName(a, b));
break;
case 'restarts-asc':
list.sort((a, b) => (a.restarts || 0) - (b.restarts || 0) || byName(a, b));
break;
case 'mem-desc':
list.sort((a, b) => (b.memMB || 0) - (a.memMB || 0) || byName(a, b));
break;
case 'mem-asc':
list.sort((a, b) => (a.memMB || 0) - (b.memMB || 0) || byName(a, b));
break;
case 'status':
default:
// Errored first, then stopped, then online; tiebreak by name
list.sort((a, b) => statusRank(a.status) - statusRank(b.status) || byName(a, b));
}
res.json({ count: list.length, processes: list });
});
app.get('/api/actions', async (req, res) => {
try {
const limit = parseInt(req.query.limit) || 50;
const { rows } = await pool.query(
'SELECT * FROM timeout_agent_log ORDER BY created_at DESC LIMIT $1', [limit]
);
res.json({ count: rows.length, actions: rows });
} catch (e) {
res.json({ count: watchState.recentActions.length, actions: watchState.recentActions });
}
});
app.post('/api/restart/:name', async (req, res) => {
const name = req.params.name;
const procs = getPM2List();
const proc = procs.find(p => p.name === name);
if (!proc) return res.status(404).json({ error: 'Process not found' });
broadcast(`MANUAL RESTART: ${name} (id:${proc.pm_id})`);
const result = restartProcess(name, proc.pm_id);
watchState.cooldowns[name] = Date.now();
watchState.totalRestarts++;
await logAction(name, proc.pm_id, 'manual_restart', 'user triggered', proc.pm2_env?.status, proc.pm2_env?.restart_time, result.success, result.error || null);
res.json({ success: result.success, error: result.error });
});
app.post('/api/toggle', (req, res) => {
watchState.enabled = !watchState.enabled;
broadcast(`Watchdog ${watchState.enabled ? 'ENABLED' : 'DISABLED'}`);
res.json({ enabled: watchState.enabled });
});
app.get('/api/stats', async (req, res) => {
try {
const today = await pool.query(
"SELECT COUNT(*) as cnt FROM timeout_agent_log WHERE action IN ('restart','hard_restart') AND created_at > NOW() - INTERVAL '24 hours'"
);
const week = await pool.query(
"SELECT COUNT(*) as cnt FROM timeout_agent_log WHERE action IN ('restart','hard_restart') AND created_at > NOW() - INTERVAL '7 days'"
);
const topOffenders = await pool.query(
"SELECT pm2_name, COUNT(*) as cnt FROM timeout_agent_log WHERE action IN ('restart','hard_restart') AND created_at > NOW() - INTERVAL '7 days' GROUP BY pm2_name ORDER BY cnt DESC LIMIT 10"
);
res.json({
restarts24h: parseInt(today.rows[0].cnt),
restarts7d: parseInt(week.rows[0].cnt),
topOffenders: topOffenders.rows
});
} catch (e) {
res.status(500).json({ error: e.message });
}
});
// ── Health Check ──
function healthHandler(req, res) {
res.json({ status: 'ok', agent: 'The Warden', port: PORT, uptime: process.uptime() });
}
app.get('/health', healthHandler);
app.get('/api/health', healthHandler);
// ── Dashboard ──
app.get('/', (req, res) => {
res.setHeader('Content-Type', 'text/html');
res.send(`<!DOCTYPE html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
<title>Timeout Agent - Process Watchdog</title>
<style>
*{margin:0;padding:0;box-sizing:border-box}
body{font-family:-apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh}
.header{background:linear-gradient(135deg,#1e293b,#0f172a);border-bottom:2px solid ${AGENT_COLOR};padding:20px 30px;display:flex;justify-content:space-between;align-items:center}
.header h1{font-size:24px;color:${AGENT_COLOR};display:flex;align-items:center;gap:10px}
.agent-avatar{width:56px;height:56px;border-radius:50%;border:2px solid #2C3539;object-fit:cover}
.agent-avatar-fallback{width:56px;height:56px;border-radius:50%;background:#2C3539;color:#fff;display:none;align-items:center;justify-content:center;font-size:22px;font-weight:700;flex-shrink:0}
.header .dot{width:12px;height:12px;border-radius:50%;background:#22c55e;animation:pulse 2s infinite}
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.4}}
.stats-bar{display:flex;gap:15px;padding:15px 30px;background:#1e293b;border-bottom:1px solid #334155;flex-wrap:wrap}
.stat{background:#0f172a;border:1px solid #334155;border-radius:8px;padding:12px 20px;min-width:140px}
.stat .label{font-size:11px;color:#94a3b8;text-transform:uppercase;letter-spacing:1px}
.stat .value{font-size:22px;font-weight:700;color:${AGENT_COLOR};margin-top:4px}
.stat .value.green{color:#22c55e}
.stat .value.red{color:#ef4444}
.stat .value.yellow{color:#f59e0b}
.main{padding:20px 30px;display:grid;grid-template-columns:1fr 1fr;gap:20px}
@media(max-width:1200px){.main{grid-template-columns:1fr}}
.card{background:#1e293b;border:1px solid #334155;border-radius:12px;overflow:hidden}
.card-header{padding:15px 20px;border-bottom:1px solid #334155;font-weight:600;font-size:14px;display:flex;justify-content:space-between;align-items:center}
.card-body{padding:15px 20px;max-height:500px;overflow-y:auto}
.proc-row{display:flex;align-items:center;gap:12px;padding:8px 0;border-bottom:1px solid #1e293b}
.proc-row:last-child{border-bottom:none}
.proc-dot{width:8px;height:8px;border-radius:50%;flex-shrink:0}
.proc-dot.online{background:#22c55e}
.proc-dot.errored{background:#ef4444}
.proc-dot.stopped{background:#94a3b8}
.proc-name{flex:1;font-size:13px;font-family:monospace}
.proc-restarts{font-size:11px;color:#94a3b8;min-width:50px;text-align:right}
.proc-restarts.warn{color:#f59e0b}
.proc-restarts.danger{color:#ef4444}
.proc-mem{font-size:11px;color:#64748b;min-width:60px;text-align:right}
.proc-btn{background:${AGENT_COLOR};color:#000;border:none;border-radius:4px;padding:3px 8px;font-size:11px;cursor:pointer;font-weight:600}
.proc-btn:hover{opacity:.8}
.log-entry{padding:6px 0;border-bottom:1px solid #0f172a;font-size:12px;display:flex;gap:8px}
.log-entry:last-child{border-bottom:none}
.log-ts{color:#64748b;font-family:monospace;font-size:11px;min-width:70px}
.log-action{font-weight:600;min-width:80px}
.log-action.restart{color:${AGENT_COLOR}}
.log-action.hard_restart{color:#ef4444}
.log-action.manual_restart{color:#8b5cf6}
.log-name{color:#e2e8f0;font-family:monospace}
.log-reason{color:#94a3b8;font-style:italic}
.toggle-btn{background:${AGENT_COLOR};color:#000;border:none;border-radius:6px;padding:8px 16px;font-weight:700;cursor:pointer;font-size:13px}
.toggle-btn:hover{opacity:.85}
.toggle-btn.off{background:#475569}
.live-log{background:#0f172a;border:1px solid #334155;border-radius:8px;padding:10px;max-height:200px;overflow-y:auto;font-family:monospace;font-size:11px;color:#94a3b8;margin-top:15px}
.live-line{padding:2px 0}
.grid-controls{display:flex;gap:10px;align-items:center;flex-wrap:wrap}
.grid-controls label{font-size:11px;color:#94a3b8;text-transform:uppercase;letter-spacing:1px}
.grid-controls select{background:#0f172a;color:#e2e8f0;border:1px solid #334155;border-radius:6px;padding:5px 8px;font-size:12px;cursor:pointer}
.grid-controls input[type=range]{width:90px;accent-color:${AGENT_COLOR}}
.grid-controls .density-val{font-size:11px;color:#64748b;font-family:monospace;min-width:24px;text-align:right}
</style></head><body>
<div class="header">
<img class="agent-avatar" src="http://45.61.58.125/agent-avatars/the-warden.png" onerror="this.style.display='none';this.nextElementSibling.style.display='flex'"><div class="agent-avatar-fallback">W</div>
<h1><div class="dot" id="dotIndicator"></div>Timeout Agent</h1>
<div style="display:flex;gap:10px;align-items:center">
<span id="statusText" style="color:#94a3b8;font-size:13px">Loading...</span>
<button class="toggle-btn" id="toggleBtn" onclick="toggleWatch()">Enabled</button>
</div>
</div>
<div class="stats-bar" id="statsBar">
<div class="stat"><div class="label">Processes</div><div class="value" id="totalProcs">-</div></div>
<div class="stat"><div class="label">Online</div><div class="value green" id="onlineProcs">-</div></div>
<div class="stat"><div class="label">Errored</div><div class="value red" id="erroredProcs">-</div></div>
<div class="stat"><div class="label">Total Checks</div><div class="value" id="totalChecks">-</div></div>
<div class="stat"><div class="label">Total Restarts</div><div class="value yellow" id="totalRestarts">-</div></div>
<div class="stat"><div class="label">Restarts (24h)</div><div class="value yellow" id="restarts24h">-</div></div>
<div class="stat"><div class="label">Last Check</div><div class="value" id="lastCheck" style="font-size:14px">-</div></div>
</div>
<div class="main">
<div class="card">
<div class="card-header">
<span>Process Status <span id="procCount" style="color:#64748b;font-weight:400"></span></span>
<div class="grid-controls">
<label for="sortSel">Sort</label>
<select id="sortSel" onchange="onSortChange()">
<option value="status">Status (errored first)</option>
<option value="name">Name A→Z</option>
<option value="name-desc">Name Z→A</option>
<option value="restarts-desc">Restarts ↓</option>
<option value="restarts-asc">Restarts ↑</option>
<option value="mem-desc">Memory ↓</option>
<option value="mem-asc">Memory ↑</option>
</select>
<label for="densSel">Density</label>
<input id="densSel" type="range" min="2" max="14" step="1" value="8" oninput="onDensityChange(this.value)">
<span class="density-val" id="densVal">8</span>
</div>
</div>
<div class="card-body" id="procList">Loading...</div>
</div>
<div class="card">
<div class="card-header">Recent Actions <span style="color:#64748b;font-weight:400">(auto-restarts)</span></div>
<div class="card-body" id="actionList">Loading...</div>
</div>
</div>
<div style="padding:0 30px 20px">
<div class="card" style="margin-top:20px">
<div class="card-header">Live Log</div>
<div class="live-log" id="liveLog"></div>
</div>
</div>
<script>
const AUTH='Basic '+btoa('admin:DWSecure2024!');
const H={headers:{Authorization:AUTH}};
// ── Grid controls: sort + density, persisted to localStorage ──
const LS_SORT='timeout-agent.proc-sort';
const LS_DENS='timeout-agent.proc-density';
let currentSort=localStorage.getItem(LS_SORT)||'status';
let currentDensity=parseInt(localStorage.getItem(LS_DENS),10);
if(!Number.isFinite(currentDensity)||currentDensity<2||currentDensity>14)currentDensity=8;
// Hydrate controls BEFORE first refresh() so the initial API call uses the saved sort
(function hydrateGridControls(){
const ss=document.getElementById('sortSel');
if(ss)ss.value=currentSort;
const ds=document.getElementById('densSel');
if(ds)ds.value=String(currentDensity);
const dv=document.getElementById('densVal');
if(dv)dv.textContent=String(currentDensity);
})();
function onSortChange(){
const ss=document.getElementById('sortSel');
currentSort=ss?ss.value:'status';
try{localStorage.setItem(LS_SORT,currentSort);}catch(e){}
refresh();
}
function onDensityChange(v){
const n=parseInt(v,10);
if(!Number.isFinite(n))return;
currentDensity=n;
try{localStorage.setItem(LS_DENS,String(n));}catch(e){}
const dv=document.getElementById('densVal');if(dv)dv.textContent=String(n);
applyDensity();
}
function applyDensity(){
// Density slider drives vertical padding on each proc row (2 = tightest, 14 = loosest)
document.querySelectorAll('#procList .proc-row').forEach(el=>{
el.style.paddingTop=currentDensity+'px';
el.style.paddingBottom=currentDensity+'px';
});
}
async function refresh(){
try{
const[st,pr,ac,stats]=await Promise.all([
fetch('/api/status',H).then(r=>r.json()),
fetch('/api/processes?sort='+encodeURIComponent(currentSort),H).then(r=>r.json()),
fetch('/api/actions?limit=30',H).then(r=>r.json()),
fetch('/api/stats',H).then(r=>r.json())
]);
document.getElementById('totalProcs').textContent=st.processes.total;
document.getElementById('onlineProcs').textContent=st.processes.online;
document.getElementById('erroredProcs').textContent=st.processes.errored;
document.getElementById('totalChecks').textContent=st.totalChecks;
document.getElementById('totalRestarts').textContent=st.totalRestarts;
document.getElementById('restarts24h').textContent=stats.restarts24h||0;
document.getElementById('lastCheck').textContent=st.lastCheck?new Date(st.lastCheck).toLocaleTimeString():'Never';
document.getElementById('statusText').textContent=st.watching?'Watching every 60s':'PAUSED';
const btn=document.getElementById('toggleBtn');
btn.textContent=st.watching?'Enabled':'Disabled';
btn.className='toggle-btn'+(st.watching?'':' off');
document.getElementById('dotIndicator').style.background=st.watching?'#22c55e':'#ef4444';
// Process list
const pl=document.getElementById('procList');
if(pr.processes.length===0){pl.innerHTML='<div style="color:#64748b">No processes found</div>';return;}
pl.innerHTML=pr.processes.map(p=>{
const dotClass=p.status==='online'?'online':p.status==='errored'?'errored':'stopped';
const rClass=p.restarts>100?'danger':p.restarts>20?'warn':'';
return '<div class="proc-row">'+
'<div class="proc-dot '+dotClass+'"></div>'+
'<div class="proc-name">'+p.name+'</div>'+
'<div class="proc-restarts '+rClass+'">'+p.restarts+'x</div>'+
'<div class="proc-mem">'+(p.memMB||0)+'MB</div>'+
(p.status!=='online'?'<button class="proc-btn" onclick="manualRestart(\\''+p.name+'\\')">Restart</button>':'')+
'</div>';
}).join('');
document.getElementById('procCount').textContent='('+pr.count+')';
applyDensity();
// Actions
const al=document.getElementById('actionList');
if(ac.actions.length===0){al.innerHTML='<div style="color:#64748b">No actions yet</div>';return;}
al.innerHTML=ac.actions.map(a=>{
const ts=a.created_at?new Date(a.created_at).toLocaleTimeString():a.ts?new Date(a.ts).toLocaleTimeString():'';
const action=a.action||'unknown';
const name=a.pm2_name||a.name||'?';
const reason=a.reason||'';
return '<div class="log-entry">'+
'<span class="log-ts">'+ts+'</span>'+
'<span class="log-action '+action+'">'+action+'</span>'+
'<span class="log-name">'+name+'</span>'+
'<span class="log-reason">'+reason+'</span></div>';
}).join('');
}catch(e){console.error(e)}
}
async function toggleWatch(){
await fetch('/api/toggle',{method:'POST',...H});
refresh();
}
async function manualRestart(name){
if(!confirm('Restart '+name+'?'))return;
await fetch('/api/restart/'+name,{method:'POST',...H});
setTimeout(refresh,2000);
}
// SSE live log
const evtSrc=new EventSource('/api/events');
evtSrc.onmessage=function(e){
try{
const d=JSON.parse(e.data);
const log=document.getElementById('liveLog');
const line=document.createElement('div');
line.className='live-line';
line.textContent=new Date(d.ts).toLocaleTimeString()+' '+d.msg;
log.appendChild(line);
log.scrollTop=log.scrollHeight;
if(log.children.length>100)log.removeChild(log.firstChild);
}catch(e){}
};
refresh();
setInterval(refresh,15000);
</script></body></html>`);
});
// ── Startup ──
async function start() {
// DB init is non-fatal: the watchdog can still restart PM2 processes even
// if Postgres is temporarily unavailable. Log actions will fall back to
// in-memory recentActions until the DB comes back.
try {
await initDB();
} catch (dbErr) {
console.error('[Timeout] DB init failed (non-fatal, running without persistent logs):', dbErr.message);
broadcast('WARNING: DB unavailable — running in degraded mode (in-memory logs only)');
}
// Seed prevRestartCounts so first check doesn't false-positive on historical restarts
try {
const seedProcs = getPM2List();
for (const p of seedProcs) {
prevRestartCounts[p.name] = p.pm2_env?.restart_time || 0;
}
broadcast(`Seeded restart counts for ${seedProcs.length} processes`);
} catch (e) {}
// Start watchdog loop
setInterval(() => {
if (watchState.enabled) checkProcesses();
}, CHECK_INTERVAL);
// Initial check after 10 seconds
setTimeout(() => {
if (watchState.enabled) checkProcesses();
}, 10000);
app.listen(PORT, '0.0.0.0', () => {
broadcast(`${AGENT_NAME} v2.0-smart running on port ${PORT} | Checking every 60s`);
broadcast(`Dashboard: http://45.61.58.125:${PORT}`);
});
}
start().catch(err => {
console.error('[Timeout] Fatal startup error:', err);
process.exit(1);
});