`monitor-health.sh`

79 lines
#!/bin/bash

# Health check monitoring script for goodquestion
# Monitors port 3003 and restarts PM2 if server is down or unresponsive

LOG_FILE="/root/WebsitesMisc/goodquestion/logs/health-monitor.log"
MAX_HANG_TIME=600  # 10 minutes in seconds
PORT=3003
APP_NAME="goodquestion"

log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

check_port() {
    lsof -i :$PORT -sTCP:LISTEN >/dev/null 2>&1
    return $?
}

check_pm2_running() {
    pm2 describe $APP_NAME >/dev/null 2>&1
    return $?
}

restart_server() {
    log_message "CRITICAL: Restarting server on port $PORT"
    pm2 restart $APP_NAME
    sleep 10
    if check_port; then
        log_message "SUCCESS: Server restarted and listening on port $PORT"
    else
        log_message "ERROR: Server restart failed, trying full restart"
        pm2 delete $APP_NAME 2>/dev/null
        cd /root/WebsitesMisc/goodquestion
        pm2 start ecosystem.config.js
        sleep 10
        if check_port; then
            log_message "SUCCESS: Full restart successful"
        else
            log_message "CRITICAL: Full restart failed!"
        fi
    fi
}

# Main monitoring loop
while true; do
    if ! check_pm2_running; then
        log_message "WARNING: PM2 app not running, starting it"
        cd /root/WebsitesMisc/goodquestion
        pm2 start ecosystem.config.js
        sleep 10
    fi

    if ! check_port; then
        log_message "WARNING: Port $PORT not listening, restarting"
        restart_server
    else
        # Check for hanging/unresponsive server
        if timeout 5 curl -s http://localhost:$PORT >/dev/null 2>&1; then
            # Server is responsive
            LAST_RESPONSE_TIME=$(date +%s)
        else
            CURRENT_TIME=$(date +%s)
            if [ -z "$LAST_RESPONSE_TIME" ]; then
                LAST_RESPONSE_TIME=$CURRENT_TIME
            fi
            HANG_DURATION=$((CURRENT_TIME - LAST_RESPONSE_TIME))

            if [ $HANG_DURATION -gt $MAX_HANG_TIME ]; then
                log_message "WARNING: Server hung for $HANG_DURATION seconds, restarting"
                restart_server
                LAST_RESPONSE_TIME=$(date +%s)
            fi
        fi
    fi

    sleep 30
done