`scripts/health-monitor.sh`

173 lines
#!/bin/bash

################################################################################
# OMEGA WATCHES - HEALTH MONITORING SCRIPT
# Continuous monitoring with auto-restart capabilities
################################################################################

set -euo pipefail

APP_NAME="omega-watches"
HEALTH_ENDPOINT="http://localhost:7500/api/health"
MAX_FAILURES=3
FAILURE_COUNT=0
CHECK_INTERVAL=30
LOG_FILE="/root/Projects/watches/logs/health-monitor.log"

# Colors
GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Logging
mkdir -p "$(dirname "$LOG_FILE")"

log() {
    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" | tee -a "$LOG_FILE"
}

log_error() {
    echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ERROR:${NC} $1" | tee -a "$LOG_FILE"
}

log_warning() {
    echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] WARNING:${NC} $1" | tee -a "$LOG_FILE"
}

# Health check function
check_health() {
    if curl -f -s --max-time 10 "$HEALTH_ENDPOINT" > /dev/null 2>&1; then
        return 0
    else
        return 1
    fi
}

# Restart application
restart_app() {
    log_error "Attempting to restart application..."

    pm2 restart $APP_NAME

    sleep 10

    if check_health; then
        log "Application restarted successfully ✓"
        FAILURE_COUNT=0
        return 0
    else
        log_error "Application restart failed!"
        return 1
    fi
}

# Send alert (placeholder for integration with alerting system)
send_alert() {
    local message=$1

    # Log to file
    echo "[ALERT] $message" >> "$LOG_FILE"

    # You can add webhook notifications here
    # curl -X POST -H 'Content-Type: application/json' \
    #   -d "{\"text\":\"$message\"}" \
    #   http://your-webhook-url
}

# Main monitoring loop
monitor() {
    log "Starting health monitoring for $APP_NAME..."
    log "Health endpoint: $HEALTH_ENDPOINT"
    log "Check interval: ${CHECK_INTERVAL}s"
    log "Max failures before restart: $MAX_FAILURES"

    while true; do
        if check_health; then
            if [ $FAILURE_COUNT -gt 0 ]; then
                log "Health check passed (recovered from $FAILURE_COUNT failures)"
            fi
            FAILURE_COUNT=0
        else
            ((FAILURE_COUNT++))
            log_warning "Health check failed (attempt $FAILURE_COUNT/$MAX_FAILURES)"

            if [ $FAILURE_COUNT -ge $MAX_FAILURES ]; then
                log_error "Maximum failures reached! Initiating restart..."
                send_alert "CRITICAL: $APP_NAME failed $MAX_FAILURES health checks. Restarting..."

                if restart_app; then
                    send_alert "INFO: $APP_NAME restarted successfully"
                else
                    send_alert "CRITICAL: $APP_NAME restart failed! Manual intervention required!"
                fi
            fi
        fi

        sleep $CHECK_INTERVAL
    done
}

# Show usage
usage() {
    echo "Usage: $0 {start|check|status}"
    echo ""
    echo "Commands:"
    echo "  start   - Start continuous monitoring (runs in background)"
    echo "  check   - Perform single health check"
    echo "  status  - Show monitoring status"
    exit 1
}

# Single health check
single_check() {
    echo "Performing health check..."

    if check_health; then
        echo "✓ Application is healthy"
        curl -s "$HEALTH_ENDPOINT" | jq '.' 2>/dev/null || curl -s "$HEALTH_ENDPOINT"
        exit 0
    else
        echo "✗ Application health check failed!"
        exit 1
    fi
}

# Show status
show_status() {
    echo "Health Monitor Status"
    echo "====================="
    echo ""
    echo "Application: $APP_NAME"
    echo "Endpoint: $HEALTH_ENDPOINT"
    echo ""

    if check_health; then
        echo "Status: ✓ HEALTHY"
        echo ""
        curl -s "$HEALTH_ENDPOINT" | jq '.' 2>/dev/null || curl -s "$HEALTH_ENDPOINT"
    else
        echo "Status: ✗ UNHEALTHY"
    fi

    echo ""
    echo "Recent log entries:"
    tail -n 10 "$LOG_FILE" 2>/dev/null || echo "No logs available"
}

# Main
case "${1:-}" in
    start)
        monitor
        ;;
    check)
        single_check
        ;;
    status)
        show_status
        ;;
    *)
        usage
        ;;
esac