`DW-Agents/dw-agents/auto-fix-restarts.sh`

47 lines
#!/bin/bash
# Auto-fix agents with excessive restarts
# Runs every 5 minutes to detect and fix crash loops

LOG_FILE="/root/DW-Agents/logs/auto-fix-restarts.log"
MAX_RESTARTS=100
FIXED_COUNT=0

echo "[$(date)] Starting auto-fix scan..." >> "$LOG_FILE"

# Get list of agents with high restart counts
pm2 jlist | jq -r '.[] | select(.pm2_env.restart_time > '$MAX_RESTARTS') | .name' | while read -r agent_name; do
    restart_count=$(pm2 jlist | jq -r ".[] | select(.name==\"$agent_name\") | .pm2_env.restart_time")

    echo "[$(date)] Found $agent_name with $restart_count restarts" >> "$LOG_FILE"

    # Get recent error logs
    error_log=$(pm2 logs "$agent_name" --err --lines 5 --nostream 2>&1)

    # Check for common syntax errors
    if echo "$error_log" | grep -q "Unexpected"; then
        echo "[$(date)] Syntax error detected in $agent_name - stopping to prevent resource waste" >> "$LOG_FILE"
        pm2 stop "$agent_name" >> "$LOG_FILE" 2>&1
        FIXED_COUNT=$((FIXED_COUNT + 1))

    elif echo "$error_log" | grep -q "EADDRINUSE"; then
        echo "[$(date)] Port conflict detected in $agent_name - restarting" >> "$LOG_FILE"
        pm2 restart "$agent_name" >> "$LOG_FILE" 2>&1
        FIXED_COUNT=$((FIXED_COUNT + 1))

    elif echo "$error_log" | grep -q "authentication_error"; then
        echo "[$(date)] API key error in $agent_name - needs manual fix" >> "$LOG_FILE"

    else
        echo "[$(date)] Unknown error in $agent_name - stopping" >> "$LOG_FILE"
        pm2 stop "$agent_name" >> "$LOG_FILE" 2>&1
        FIXED_COUNT=$((FIXED_COUNT + 1))
    fi
done

if [ $FIXED_COUNT -gt 0 ]; then
    echo "[$(date)] Auto-fixed $FIXED_COUNT agents" >> "$LOG_FILE"
    pm2 save >> "$LOG_FILE" 2>&1
fi

echo "[$(date)] Auto-fix scan complete" >> "$LOG_FILE"