← back to Designer Wallcoverings
DW-Agents/dw-agents/agent-health-monitor.sh
203 lines
#!/bin/bash
#
# DW-Agents Health Monitor
# Runs every 5 minutes via cron to ensure all agents are:
# - Running and accessible
# - Have firewall ports open
# - No port conflicts
# - Auto-restart if down
#
LOG_FILE="/tmp/agent-health-monitor.log"
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
echo "[$TIMESTAMP] Starting agent health check..." >> "$LOG_FILE"
# Agent registry with ports
declare -A AGENTS=(
[9882]="accounting-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-accounting"
[9881]="marketing-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-marketing"
[9878]="legal-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-legal-team"
[9880]="purchasing-office-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-purchasing-office"
[7238]="shopify-store-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-shopify-store"
[9883]="trend-research-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-trend-research"
[9892]="security-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-security"
[9879]="digital-samples-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-digital-samples"
[9890]="new-client-signup-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-new-client-signup"
[9894]="skills-manager-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-skills-manager"
[9111]="agents-viewer.ts|/root/Projects/Designer-Wallcoverings/DW-Agents"
[9900]="task-orchestrator-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-task-orchestrator"
[9885]="todays-highlights-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-todays-highlights"
[9886]="needs-attention-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-needs-attention"
[9889]="completed-tasks-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-completed-tasks"
[9884]="zendesk-chat-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-zendesk-chat"
[9888]="server-uptime-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-server-uptime"
[7239]="log-monitor-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-log-monitor"
[9887]="parallel-processes-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-parallel-processes"
[7121]="ceo-dashboard-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-executive-ceo"
[7122]="cfo-dashboard-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-executive-cfo"
[7123]="vp-ops-dashboard-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-executive-vp-ops"
[7124]="coo-dashboard-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-executive-coo"
[9895]="crash-loop-resolver.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-crash-loop-resolver"
[9893]="master-hub.ts|/root/Projects/Designer-Wallcoverings/DW-Agents"
[7240]="ui-manager-agent.ts|/root/Projects/Designer-Wallcoverings/DW-Agents/agent-ui-manager"
)
# Function to check if port is listening
is_port_listening() {
local port=$1
netstat -tlnp 2>/dev/null | grep -q ":${port}.*LISTEN"
return $?
}
# Function to check if port responds to HTTP
is_port_responding() {
local port=$1
local response=$(curl -s -o /dev/null -w "%{http_code}" --max-time 3 "http://localhost:${port}/" 2>/dev/null)
[[ "$response" =~ ^(200|302|301)$ ]]
return $?
}
# Function to check if firewall port is open
is_firewall_open() {
local port=$1
sudo ufw status | grep -q "^${port}/tcp.*ALLOW"
return $?
}
# Function to open firewall port
open_firewall() {
local port=$1
echo " [FIREWALL] Opening port ${port}..." >> "$LOG_FILE"
sudo ufw allow ${port}/tcp >> "$LOG_FILE" 2>&1
}
# PM2 process name mapping by port
declare -A PM2_NAMES=(
[9882]="dw-accounting"
[9881]="dw-marketing"
[9878]="dw-legal-team"
[9880]="dw-purchasing-office"
[7238]="dw-shopify-store"
[9883]="dw-trend-research"
[9892]="dw-security"
[9879]="dw-digital-samples"
[9890]="dw-new-client-signup"
[9894]="dw-skills-manager"
[9900]="dw-task-orchestrator"
[9885]="dw-todays-highlights"
[9886]="dw-needs-attention"
[9889]="dw-completed-tasks"
[9884]="dw-zendesk-chat"
[9888]="dw-server-uptime"
[7239]="dw-log-monitor"
[9887]="dw-in-parallel"
[7121]="dw-executive-ceo"
[7122]="dw-executive-cfo"
[7123]="dw-executive-vp-ops"
[7124]="dw-executive-coo"
[9895]="dw-crash-loop-resolver"
[9893]="dw-master-hub"
[7240]="dw-ui-manager"
)
# Function to restart agent
restart_agent() {
local port=$1
local script=$2
local dir=$3
echo " [RESTART] Restarting agent on port ${port}..." >> "$LOG_FILE"
# Check if agent is managed by PM2
if [[ -n "${PM2_NAMES[$port]}" ]]; then
local pm2_name="${PM2_NAMES[$port]}"
echo " [PM2] Using PM2 to restart ${pm2_name}..." >> "$LOG_FILE"
pm2 restart "${pm2_name}" >> "$LOG_FILE" 2>&1
sleep 3
if is_port_listening ${port}; then
echo " [SUCCESS] Agent on port ${port} restarted successfully via PM2" >> "$LOG_FILE"
return 0
else
echo " [ERROR] Failed to restart agent on port ${port} via PM2" >> "$LOG_FILE"
return 1
fi
else
# Fallback to manual restart for non-PM2 agents
echo " [MANUAL] Using manual restart for port ${port}..." >> "$LOG_FILE"
# Kill existing process
lsof -ti:${port} | xargs kill -9 2>/dev/null
pkill -f "${script}" 2>/dev/null
sleep 2
# Start agent
cd "${dir}"
npx tsx "${script}" > "/tmp/agent-${port}.log" 2>&1 &
sleep 3
if is_port_listening ${port}; then
echo " [SUCCESS] Agent on port ${port} restarted successfully" >> "$LOG_FILE"
return 0
else
echo " [ERROR] Failed to restart agent on port ${port}" >> "$LOG_FILE"
return 1
fi
fi
}
# Check each agent
TOTAL_AGENTS=${#AGENTS[@]}
HEALTHY_AGENTS=0
RESTARTED_AGENTS=0
FIREWALL_FIXED=0
for port in "${!AGENTS[@]}"; do
IFS='|' read -r script dir <<< "${AGENTS[$port]}"
echo "" >> "$LOG_FILE"
echo "Checking port ${port} (${script})..." >> "$LOG_FILE"
# Check if firewall is open
if ! is_firewall_open ${port}; then
echo " [WARNING] Firewall closed for port ${port}" >> "$LOG_FILE"
open_firewall ${port}
((FIREWALL_FIXED++))
fi
# Check if port is listening
if ! is_port_listening ${port}; then
echo " [DOWN] Port ${port} not listening" >> "$LOG_FILE"
restart_agent ${port} "${script}" "${dir}"
((RESTARTED_AGENTS++))
elif ! is_port_responding ${port}; then
echo " [UNHEALTHY] Port ${port} not responding to HTTP" >> "$LOG_FILE"
restart_agent ${port} "${script}" "${dir}"
((RESTARTED_AGENTS++))
else
echo " [OK] Port ${port} healthy" >> "$LOG_FILE"
((HEALTHY_AGENTS++))
fi
done
# Summary
echo "" >> "$LOG_FILE"
echo "[$TIMESTAMP] Health Check Summary:" >> "$LOG_FILE"
echo " Total Agents: ${TOTAL_AGENTS}" >> "$LOG_FILE"
echo " Healthy: ${HEALTHY_AGENTS}" >> "$LOG_FILE"
echo " Restarted: ${RESTARTED_AGENTS}" >> "$LOG_FILE"
echo " Firewall Fixed: ${FIREWALL_FIXED}" >> "$LOG_FILE"
echo "========================================" >> "$LOG_FILE"
# Exit with error if any agents were restarted (for monitoring)
if [ ${RESTARTED_AGENTS} -gt 0 ]; then
exit 1
else
exit 0
fi