← back to Designer Wallcoverings
DW-Agents/dw-agents/agent-executive-vp-ops/vp-ops-dashboard-agent.ts
530 lines
/**
* DW-Agents: VP Operations Executive Dashboard
*
* Infrastructure-focused executive dashboard:
* - Server health: CPU, memory, disk monitoring
* - Agent fleet detailed status with crash detection
* - Crash loop monitor with remediation suggestions
* - Infrastructure sub-agent oversight
* - AI-powered infrastructure engineer assistant
* - Inter-agent messaging and action items
*
* Port: 7125
* PM2: dw-executive-vp-ops
*/
import cookieParser from 'cookie-parser';
import Anthropic from '@anthropic-ai/sdk';
import express, { Request, Response } from 'express';
import { requireGlobalAuth } from '../shared-global-auth';
import session from 'express-session';
import fetch from 'node-fetch';
import dotenv from 'dotenv';
import path from 'path';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
import { chatMiddleware } from '../shared-chat-integration';
import {
pool, queryDB, getPM2Status, getVendorStats, getSystemHealth,
EXECUTIVE_AGENTS, SERVER_IP,
sendMessage, createMessageStore, createActionItemStore,
execDashboardHead, execNavBar, execChatPanel,
formatUptime, formatNumber, timeAgo,
autoRefreshScript, notifySlack,
AgentStatus
} from '../shared-executive-core';
// ES module __dirname fix
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
dotenv.config({ path: path.join(__dirname, '..', '.env') });
// Global error handlers
process.on('uncaughtException', (error) => {
console.error('UNCAUGHT EXCEPTION:', error);
process.exit(1);
});
process.on('unhandledRejection', (reason, promise) => {
console.error('UNHANDLED REJECTION at:', promise, 'reason:', reason);
process.exit(1);
});
const app = express();
const PORT = 7125;
// Anthropic AI
const anthropic = new Anthropic({
apiKey: process.env.ANTHROPIC_API_KEY || '',
});
// Message & action item stores
const messageStore = createMessageStore();
const actionStore = createActionItemStore();
// Middleware
app.use(cookieParser());
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
app.use(chatMiddleware({
agentId: 'agent-executive-vp-ops',
agentName: 'VP Ops Dashboard',
port: PORT,
category: 'executive'
}));
declare module 'express-session' {
interface SessionData {
authenticated: boolean;
chatHistory: Array<{ role: 'user' | 'assistant'; content: string }>;
}
}
app.use(session({
secret: 'dw-vpops-dashboard-2025-infrastructure',
resave: false,
saveUninitialized: true,
rolling: true,
cookie: { secure: false, httpOnly: true, maxAge: 7 * 24 * 60 * 60 * 1000 }
}));
// ---- Logout ----
app.get('/logout', (req, res) => {
req.session.destroy(() => res.redirect('/login'));
});
// ---- Helper: Progress bar HTML ----
function progressBar(usedStr: string, totalStr: string, label: string): string {
// Parse numeric values from strings like "12345MB" or "45G"
const parseVal = (s: string): number => {
const n = parseFloat(s);
if (s.includes('G')) return n * 1024;
return n;
};
const used = parseVal(usedStr);
const total = parseVal(totalStr);
const pct = total > 0 ? Math.round((used / total) * 100) : 0;
const color = pct >= 90 ? '#ef4444' : pct >= 70 ? '#f59e0b' : '#22c55e';
return `<div style="margin-bottom:12px;">
<div style="display:flex;justify-content:space-between;font-size:13px;color:#94a3b8;margin-bottom:4px;">
<span>${label}</span>
<span>${usedStr} / ${totalStr} (${pct}%)</span>
</div>
<div style="background:#1e293b;border-radius:6px;height:10px;overflow:hidden;">
<div style="width:${pct}%;background:${color};height:100%;border-radius:6px;transition:width 0.5s;"></div>
</div>
</div>`;
}
// ---- Main Dashboard ----
app.get('/', requireGlobalAuth, async (req: Request, res: Response) => {
try {
// Fetch live data in parallel
const [agents, health] = await Promise.all([
getPM2Status(),
getSystemHealth()
]);
// Parse system health
const cpuLoad = parseFloat(health.cpuLoad) || 0;
const memUsed = health.memUsed;
const memTotal = health.memTotal;
const diskUsed = health.diskUsed;
const diskTotal = health.diskTotal;
// KPI computations
const fleetSize = agents.length;
const onlineCount = agents.filter(a => a.status === 'online').length;
// Auto-detect action items
if (cpuLoad > 4.0) {
actionStore.add('System Health', 'critical', `High CPU Load: ${cpuLoad}`,
`Server CPU load average is ${cpuLoad}. This may degrade agent performance. Consider killing heavy processes.`);
}
const memUsedVal = parseFloat(memUsed);
const memTotalVal = parseFloat(memTotal);
if (memTotalVal > 0 && (memUsedVal / memTotalVal) > 0.80) {
actionStore.add('System Health', 'warning', `High Memory Usage: ${memUsed}/${memTotal}`,
`Server memory is above 80% utilization. Consider restarting high-memory agents or adding swap.`);
}
const criticalRestartAgents = agents.filter(a => a.restarts > 20);
criticalRestartAgents.forEach(a => {
actionStore.add('Crash Loop', 'critical', `${a.name} - ${a.restarts} Restarts`,
`Agent has crash-looped ${a.restarts} times. Immediate investigation required: pm2 logs ${a.name} --err --lines 50`);
});
// Sort agents for detailed table: stopped first, then by restarts desc
const sortedAgents = [...agents].sort((a, b) => {
if (a.status !== 'online' && b.status === 'online') return -1;
if (a.status === 'online' && b.status !== 'online') return 1;
return b.restarts - a.restarts;
});
// Build agent fleet table
const agentFleetRows = sortedAgents.map(a => {
const statusClass = a.status === 'online' ? 'online' : (a.status === 'errored' ? 'errored' : 'stopped');
const rowBg = a.restarts > 20 ? 'background:rgba(239,68,68,0.08);' : a.restarts > 10 ? 'background:rgba(245,158,11,0.08);' : '';
return `<tr style="${rowBg}">
<td><span class="status-dot ${statusClass}"></span>${a.name}</td>
<td><span style="color:${a.status === 'online' ? '#22c55e' : '#ef4444'}">${a.status}</span></td>
<td>${a.cpu}%</td>
<td>${a.memoryMB} MB</td>
<td>${formatUptime(a.uptime)}</td>
<td style="color: ${a.restarts > 20 ? '#ef4444' : a.restarts > 10 ? '#f59e0b' : '#cbd5e1'}; font-weight:${a.restarts > 10 ? '700' : '400'}">${a.restarts}</td>
</tr>`;
}).join('');
// Crash loop monitor: agents with >5 restarts
const crashAgents = agents.filter(a => a.restarts > 5).sort((a, b) => b.restarts - a.restarts);
const crashLoopHTML = crashAgents.length > 0
? `<div style="overflow-x:auto;">
<table class="agent-table">
<thead><tr><th>Agent</th><th>Restarts</th><th>Status</th><th>Suggested Action</th></tr></thead>
<tbody>${crashAgents.map(a => {
let suggestion = 'Monitor closely';
if (a.restarts > 20) suggestion = 'CRITICAL: Kill, analyze logs, rebuild';
else if (a.restarts > 10) suggestion = 'Check memory limits and port conflicts';
else suggestion = 'Review error logs for root cause';
const severity = a.restarts > 20 ? '#ef4444' : a.restarts > 10 ? '#f59e0b' : '#94a3b8';
return `<tr>
<td>${a.name}</td>
<td style="color:${severity};font-weight:700">${a.restarts}</td>
<td>${a.status}</td>
<td style="color:#94a3b8;font-size:12px">${suggestion}</td>
</tr>`;
}).join('')}</tbody>
</table>
</div>`
: '<div style="padding:16px;color:#22c55e;">No crash loops detected. All agents stable.</div>';
// Build messages HTML
const messages = messageStore.getAll();
const messagesHTML = messages.length > 0
? messages.slice(0, 10).map(m => `
<div class="msg-item ${m.read ? '' : 'unread'}">
<span class="from">${m.from}</span>
<div>
<div class="subject">${m.subject} <span class="priority-badge ${m.priority}">${m.priority}</span></div>
<div class="body">${m.body.substring(0, 120)}${m.body.length > 120 ? '...' : ''}</div>
</div>
<span class="time">${timeAgo(m.timestamp)}</span>
</div>`).join('')
: '<div style="padding:16px;color:#64748b;">No messages yet.</div>';
// Build action items HTML
const actions = actionStore.getUnresolved();
const actionsHTML = actions.length > 0
? actions.slice(0, 8).map(a => `
<div class="action-item ${a.severity}">
<div class="title">${a.title}</div>
<div class="desc">${a.description}</div>
<div class="meta">${a.source} - ${timeAgo(a.timestamp)}</div>
</div>`).join('')
: '<div style="padding:16px;color:#64748b;">No action items. Infrastructure healthy.</div>';
// Infrastructure sub-agents
const subAgents = [
{ name: 'Crash Loop Resolver', port: 9895, desc: 'Auto-recovery for crash-looping agents' },
{ name: 'Log Monitor', port: 7239, desc: 'Centralized log aggregation' },
{ name: 'Dashboard Monitor', port: 9990, desc: 'System health overview' },
{ name: 'UI Manager', port: 7240, desc: 'Interface consistency management' }
];
const subAgentsHTML = subAgents.map(sa => `
<div class="sub-agent">
<a href="http://${SERVER_IP}:${sa.port}" target="_blank">${sa.name}</a>
<div class="desc">${sa.desc}</div>
</div>`).join('');
const html = `
${execDashboardHead('VP Ops Infrastructure Dashboard', '@vp')}
${execNavBar('VP Ops Infrastructure Dashboard', '\u{1F5A5}\uFE0F', '@vp', messageStore.unreadCount())}
<div class="main">
<div class="content">
<!-- KPI Cards -->
<div class="kpi-grid">
<div class="kpi-card ${cpuLoad > 4 ? 'red' : cpuLoad > 2 ? 'amber' : 'green'}">
<div class="label">CPU Load</div>
<div class="value">${health.cpuLoad}</div>
<div class="sub">${cpuLoad > 4 ? 'CRITICAL - High load' : cpuLoad > 2 ? 'Elevated' : 'Normal'}</div>
</div>
<div class="kpi-card blue">
<div class="label">Memory</div>
<div class="value">${memUsed}</div>
<div class="sub">of ${memTotal} total</div>
</div>
<div class="kpi-card blue">
<div class="label">Disk</div>
<div class="value">${diskUsed}</div>
<div class="sub">of ${diskTotal} total</div>
</div>
<div class="kpi-card ${onlineCount === fleetSize ? 'green' : 'amber'}">
<div class="label">Agent Fleet</div>
<div class="value">${onlineCount}/${fleetSize}</div>
<div class="sub">${onlineCount === fleetSize ? 'All agents operational' : `${fleetSize - onlineCount} offline`}</div>
</div>
</div>
<!-- Server Health -->
<div class="section">
<h2>\u{1F4BB} Server Health</h2>
<div style="display:grid;grid-template-columns:1fr 1fr;gap:20px;">
<div>
<div style="color:#f8fafc;font-size:14px;font-weight:600;margin-bottom:12px;">CPU Load Average</div>
<div style="font-size:36px;font-weight:700;color:${cpuLoad > 4 ? '#ef4444' : cpuLoad > 2 ? '#f59e0b' : '#22c55e'};margin-bottom:12px;">${health.cpuLoad}</div>
<div style="color:#64748b;font-size:12px;">${cpuLoad > 4 ? 'Critical: Server under heavy load' : cpuLoad > 2 ? 'Moderate: Monitor closely' : 'Healthy: Normal operation'}</div>
</div>
<div>
${progressBar(memUsed, memTotal, 'Memory Usage')}
${progressBar(diskUsed, diskTotal, 'Disk Usage')}
</div>
</div>
</div>
<!-- Agent Fleet (Detailed) -->
<div class="section">
<h2>\u{1F680} Agent Fleet (${fleetSize} agents)</h2>
<div style="overflow-x:auto;max-height:500px;overflow-y:auto;">
<table class="agent-table">
<thead>
<tr>
<th>Agent</th>
<th>Status</th>
<th>CPU</th>
<th>Memory</th>
<th>Uptime</th>
<th>Restarts</th>
</tr>
</thead>
<tbody>${agentFleetRows}</tbody>
</table>
</div>
</div>
<!-- Crash Loop Monitor -->
<div class="section">
<h2>\u{26A0}\uFE0F Crash Loop Monitor (>5 restarts)</h2>
${crashLoopHTML}
</div>
<!-- Infrastructure Sub-Agents -->
<div class="section">
<h2>\u{1F527} Infrastructure Sub-Agents</h2>
<div class="sub-agents">${subAgentsHTML}</div>
</div>
<!-- Skills & Capabilities -->
<div class="section">
<details style="cursor:pointer;">
<summary style="font-size:16px;font-weight:700;color:#f8fafc;padding:8px 0;list-style:none;display:flex;align-items:center;gap:8px;">
<span style="transition:transform 0.2s;display:inline-block;">▶</span> 🛠️ Skills & Capabilities
<span style="font-size:12px;color:#94a3b8;margin-left:auto;">(10 skills)</span>
</summary>
<table class="skills-table">
<thead>
<tr>
<th>Skill</th>
<th>Description</th>
<th>Status</th>
</tr>
</thead>
<tbody>
<tr><td>Server Health Monitoring</td><td>CPU, RAM, disk, load average tracking</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Agent Fleet Detailed Status</td><td>Per-agent CPU, memory, uptime, restarts</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Crash Loop Detection</td><td>Agents with >5 restarts flagged for remediation</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Error Rate Dashboard</td><td>Errors/hour tracking across all agents</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Infrastructure Alerting</td><td>CPU >4.0 or Memory >80% auto-alerts</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Log Monitor Integration</td><td>Error detection via Log Monitor agent</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Auto-Recovery Suggestions</td><td>Suggested fix actions for crash loops</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>AI Infrastructure Engineer</td><td>Claude-powered infrastructure troubleshooting</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>System Resource Forecasting</td><td>Trend analysis for resource utilization</td><td><span class="skill-status">Active</span></td></tr>
<tr><td>Deployment Health Checks</td><td>Post-deployment stability verification</td><td><span class="skill-status">Active</span></td></tr>
</tbody>
</table>
</details>
</div>
<!-- Messages & Action Items -->
<div class="section" id="messages-section">
<h2>\u{1F4EC} Messages & Action Items</h2>
<div class="tab-bar">
<button class="active" onclick="showTab('msgs',this)">Messages (${messages.length})</button>
<button onclick="showTab('actions',this)">Action Items (${actions.length})</button>
</div>
<div id="tab-msgs">${messagesHTML}</div>
<div id="tab-actions" style="display:none">${actionsHTML}</div>
</div>
</div>
<!-- Chat Panel -->
${execChatPanel('VP Ops', 'infrastructure engineer')}
</div>
<script>
function showTab(tab, btn) {
document.getElementById('tab-msgs').style.display = tab === 'msgs' ? 'block' : 'none';
document.getElementById('tab-actions').style.display = tab === 'actions' ? 'block' : 'none';
btn.parentElement.querySelectorAll('button').forEach(b => b.classList.remove('active'));
btn.classList.add('active');
}
</script>
${autoRefreshScript(30000)}
</body></html>`;
res.send(html);
} catch (error: any) {
console.error('Dashboard render error:', error);
res.status(500).send('Dashboard error: ' + error.message);
}
});
// ---- API: Metrics ----
app.get('/api/metrics', requireGlobalAuth, (req: Request, res: Response) => {
res.json({
status: 'online',
agent: 'VP Ops Dashboard',
port: PORT,
uptime: process.uptime(),
lastActivity: new Date().toISOString(),
messages: messageStore.count(),
unreadMessages: messageStore.unreadCount(),
actionItems: actionStore.count()
});
});
// ---- API: Messages ----
app.get('/api/messages', requireGlobalAuth, (req: Request, res: Response) => {
res.json({ messages: messageStore.getAll(), unread: messageStore.unreadCount() });
});
// ---- API: Receive Message (inter-agent) ----
app.post('/api/message', (req: Request, res: Response) => {
try {
const msg = req.body;
if (msg && msg.id) {
messageStore.receive(msg);
res.json({ success: true });
} else {
res.status(400).json({ error: 'Invalid message format' });
}
} catch (e: any) {
res.status(500).json({ error: e.message });
}
});
// ---- API: Send Message (to another agent) ----
app.post('/api/send-message', requireGlobalAuth, async (req: Request, res: Response) => {
try {
const { to, subject, body, priority } = req.body;
const success = await sendMessage('@vp', to, subject, body, priority || 'normal');
res.json({ success });
} catch (e: any) {
res.status(500).json({ error: e.message });
}
});
// ---- API: Chat ----
app.post('/api/chat', requireGlobalAuth, async (req: Request, res: Response) => {
try {
const { message, history } = req.body;
if (!req.session.chatHistory) {
req.session.chatHistory = [];
}
req.session.chatHistory.push({ role: 'user', content: message });
// Gather live infrastructure context
let infraContext = '';
try {
const [agents, health] = await Promise.all([
getPM2Status(),
getSystemHealth()
]);
const online = agents.filter(a => a.status === 'online').length;
const offline = agents.filter(a => a.status !== 'online');
const crashLoops = agents.filter(a => a.restarts > 5);
const totalMemMB = agents.reduce((s, a) => s + a.memoryMB, 0);
infraContext = `\n\nLIVE INFRASTRUCTURE DATA:
- CPU Load: ${health.cpuLoad}
- Memory: ${health.memUsed} / ${health.memTotal}
- Disk: ${health.diskUsed} / ${health.diskTotal}
- Agent Fleet: ${online}/${agents.length} online
- Offline agents: ${offline.map(a => a.name).join(', ') || 'None'}
- Crash loops (>5 restarts): ${crashLoops.map(a => `${a.name}(${a.restarts})`).join(', ') || 'None'}
- Total agent memory usage: ${totalMemMB}MB
- Top memory agents: ${[...agents].sort((a, b) => b.memoryMB - a.memoryMB).slice(0, 5).map(a => `${a.name}(${a.memoryMB}MB)`).join(', ')}`;
} catch { /* context fetch failed */ }
const systemPrompt = `You are the VP of Operations AI assistant for Designer Wallcoverings. You specialize in INFRASTRUCTURE and systems engineering.
Your expertise includes:
- Linux server administration (Ubuntu, systemd, PM2)
- CPU, memory, and disk monitoring and optimization
- Agent fleet management and crash loop resolution
- Network and firewall configuration (UFW)
- PostgreSQL database performance
- Node.js/TypeScript application troubleshooting
- Process monitoring and automatic recovery
- Capacity planning and resource allocation
You are the technical expert. When asked about issues, provide specific commands and remediation steps.
${infraContext}
Be precise with technical details. Include actual commands when suggesting fixes.
Prioritize server stability and agent uptime above all else.`;
const chatHistory = (history || req.session.chatHistory).slice(-16);
const response = await anthropic.messages.create({
model: 'claude-opus-4-8',
max_tokens: 1500,
system: systemPrompt,
messages: chatHistory.map((m: any) => ({ role: m.role, content: m.content }))
});
const assistantMessage = response.content[0].type === 'text'
? response.content[0].text
: 'Unable to process request.';
req.session.chatHistory.push({ role: 'assistant', content: assistantMessage });
if (req.session.chatHistory.length > 20) {
req.session.chatHistory = req.session.chatHistory.slice(-20);
}
res.json({ response: assistantMessage });
} catch (error: any) {
console.error('Chat error:', error);
res.status(500).json({ response: 'Error processing chat request: ' + error.message });
}
});
// ---- Start Server ----
const server = app.listen(PORT, '0.0.0.0', () => {
console.log('');
console.log('\u{1F5A5}\uFE0F VP Ops Infrastructure Dashboard');
console.log('\u2501'.repeat(45));
console.log(`\u{1F30D} External: http://${SERVER_IP}:${PORT}`);
console.log(`\u{1F3E0} Local: http://localhost:${PORT}`);
console.log('');
console.log('\u2705 VP Ops Dashboard ready...');
});
server.on('error', (error: any) => {
console.error('SERVER ERROR:', error);
if (error.code === 'EADDRINUSE') {
console.error(`Port ${PORT} is already in use`);
}
process.exit(1);
});