`scripts/devops/health-monitor.sh`

264 lines
#!/bin/bash
#
# LUXVAULT Health Monitor
# Monitors application health and sends alerts on failures
# Run every 5 minutes via cron
#

set -euo pipefail

PROJECT_DIR="/root/Projects/handbag-auth-nextjs"
LOG_FILE="$PROJECT_DIR/logs/health-monitor.log"
ALERT_LOG="$PROJECT_DIR/logs/alerts.log"
STATE_FILE="$PROJECT_DIR/.health-state"

# Configuration
HEALTH_URL="http://45.61.58.125:7500/api/status"
API_URL="http://45.61.58.125:7500/api/auctions"
MAX_RESPONSE_TIME=5000  # milliseconds
MAX_ERROR_RATE=5        # percentage
MIN_UPTIME=60           # seconds
SLACK_WEBHOOK="${SLACK_WEBHOOK_URL:-}"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

# Logging function
log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

alert() {
    local severity="$1"
    local message="$2"
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] [$severity] $message" | tee -a "$ALERT_LOG"

    # Send to Slack if webhook configured
    if [ -n "$SLACK_WEBHOOK" ]; then
        curl -X POST "$SLACK_WEBHOOK" \
            -H 'Content-Type: application/json' \
            -d "{\"text\":\"🚨 LUXVAULT Alert [$severity]: $message\"}" \
            2>/dev/null || true
    fi
}

# Check if service is running
check_pm2_status() {
    if ! pm2 status auction-viewer | grep -q "online"; then
        alert "CRITICAL" "auction-viewer is not running in PM2"
        return 1
    fi
    return 0
}

# Check HTTP health endpoint
check_health_endpoint() {
    local response
    local http_code
    local response_time

    response=$(curl -s -w "\n%{http_code}\n%{time_total}" --max-time 10 "$HEALTH_URL" 2>/dev/null || echo "")

    if [ -z "$response" ]; then
        alert "CRITICAL" "Health endpoint unreachable at $HEALTH_URL"
        return 1
    fi

    http_code=$(echo "$response" | tail -2 | head -1)
    response_time=$(echo "$response" | tail -1)
    response_body=$(echo "$response" | head -n -2)

    if [ "$http_code" != "200" ]; then
        alert "ERROR" "Health check returned HTTP $http_code"
        return 1
    fi

    # Check response time (convert to milliseconds)
    response_time_ms=$(echo "$response_time * 1000 / 1" | bc)
    if [ "$response_time_ms" -gt "$MAX_RESPONSE_TIME" ]; then
        alert "WARNING" "Slow response time: ${response_time_ms}ms (threshold: ${MAX_RESPONSE_TIME}ms)"
    fi

    # Parse JSON response - check for success field
    local success=$(echo "$response_body" | grep -o '"success":true' || echo "")

    if [ -z "$success" ]; then
        alert "WARNING" "Health check returned non-success response"
        log "Health response: $response_body"
    fi

    return 0
}

# Check API functionality
check_api_endpoint() {
    local response
    local http_code

    response=$(curl -s -w "\n%{http_code}" --max-time 10 "$API_URL?limit=1" 2>/dev/null || echo "")

    if [ -z "$response" ]; then
        alert "CRITICAL" "API endpoint unreachable at $API_URL"
        return 1
    fi

    http_code=$(echo "$response" | tail -1)

    if [ "$http_code" != "200" ]; then
        alert "ERROR" "API check returned HTTP $http_code"
        return 1
    fi

    # Check if response contains expected data
    if ! echo "$response" | grep -q '"success":true'; then
        alert "ERROR" "API response does not indicate success"
        return 1
    fi

    return 0
}

# Check database size
check_database_size() {
    local db_path="$PROJECT_DIR/data/auction-history/auctions.db"

    if [ ! -f "$db_path" ]; then
        alert "CRITICAL" "Database file not found at $db_path"
        return 1
    fi

    local db_size=$(stat -f%z "$db_path" 2>/dev/null || stat -c%s "$db_path" 2>/dev/null)
    local db_size_mb=$((db_size / 1024 / 1024))

    if [ "$db_size_mb" -gt 500 ]; then
        alert "WARNING" "Database size is ${db_size_mb}MB (consider optimization)"
    fi

    log "Database size: ${db_size_mb}MB"
    return 0
}

# Check scraper execution
check_scraper_execution() {
    local log_path="$PROJECT_DIR/data/auction-history/scraper.log"

    if [ ! -f "$log_path" ]; then
        alert "WARNING" "Scraper log file not found"
        return 1
    fi

    # Check last scraper run (should run daily at 6am)
    local last_run=$(tail -100 "$log_path" | grep "DAILY SCRAPE COMPLETED" | tail -1 || echo "")

    if [ -z "$last_run" ]; then
        # Check if we're past 7am and scraper hasn't run today
        local hour=$(date +%H)
        if [ "$hour" -gt 7 ]; then
            alert "WARNING" "Scraper may not have completed today"
        fi
    fi

    # Check for errors in recent logs
    local error_count=$(tail -100 "$log_path" | grep -c "ERROR" || true)
    if [ "$error_count" -gt 5 ]; then
        alert "WARNING" "High error count in scraper logs: $error_count errors"
    fi

    return 0
}

# Check disk space
check_disk_space() {
    local usage=$(df "$PROJECT_DIR" | tail -1 | awk '{print $5}' | sed 's/%//')

    if [ "$usage" -gt 90 ]; then
        alert "CRITICAL" "Disk usage at ${usage}%"
        return 1
    elif [ "$usage" -gt 80 ]; then
        alert "WARNING" "Disk usage at ${usage}%"
    fi

    log "Disk usage: ${usage}%"
    return 0
}

# Auto-restart if needed
auto_restart() {
    log "Attempting auto-restart of auction-viewer..."

    if pm2 restart auction-viewer; then
        alert "INFO" "Successfully restarted auction-viewer"
        sleep 5

        # Verify it's actually running
        if check_health_endpoint; then
            log "Service confirmed healthy after restart"
            return 0
        else
            alert "CRITICAL" "Service still unhealthy after restart"
            return 1
        fi
    else
        alert "CRITICAL" "Failed to restart auction-viewer via PM2"
        return 1
    fi
}

# Main health check sequence
main() {
    log "========================================="
    log "Starting health check"

    local checks_passed=0
    local checks_failed=0
    local needs_restart=false

    # Run all checks
    if check_pm2_status; then
        ((checks_passed++))
    else
        ((checks_failed++))
        needs_restart=true
    fi

    if check_health_endpoint; then
        ((checks_passed++))
    else
        ((checks_failed++))
        needs_restart=true
    fi

    if check_api_endpoint; then
        ((checks_passed++))
    else
        ((checks_failed++))
    fi

    check_database_size && ((checks_passed++)) || ((checks_failed++))
    check_scraper_execution && ((checks_passed++)) || ((checks_failed++))
    check_disk_space && ((checks_passed++)) || ((checks_failed++))

    # Report results
    log "Health check complete: $checks_passed passed, $checks_failed failed"

    # Save state
    echo "{\"timestamp\":\"$(date -Iseconds)\",\"passed\":$checks_passed,\"failed\":$checks_failed}" > "$STATE_FILE"

    # Auto-restart if critical issues detected
    if [ "$needs_restart" = true ]; then
        alert "INFO" "Critical issues detected, attempting auto-restart"
        auto_restart
    fi

    log "========================================="

    # Exit with error if any checks failed
    [ "$checks_failed" -eq 0 ]
}

# Run main function
main