`scripts/devops/health-check.sh`

174 lines
#!/bin/bash
# Comprehensive Health Check Script for Auction Viewer
# Runs every 5 minutes via cron to monitor application health

set -euo pipefail

# Configuration
SERVICE_NAME="auction-viewer"
SERVICE_URL="http://45.61.58.125:7500"
HEALTH_ENDPOINT="${SERVICE_URL}/health"
READY_ENDPOINT="${SERVICE_URL}/ready"
LOG_FILE="/root/Projects/handbag-auth-nextjs/logs/health-check.log"
METRICS_FILE="/root/Projects/handbag-auth-nextjs/data/auction-history/metrics.log"
ALERT_EMAIL=""  # Set email for alerts
MAX_RESPONSE_TIME=5000  # milliseconds
MAX_ERROR_RATE=10  # percentage

# Create log directory if needed
mkdir -p "$(dirname "${LOG_FILE}")"

# Logging function
log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "${LOG_FILE}"
}

# Metrics logging
log_metrics() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> "${METRICS_FILE}"
}

# Alert function (can be extended to send emails/Slack notifications)
send_alert() {
    local message=$1
    log "ALERT: ${message}"

    # Uncomment to enable Slack notifications
    # curl -X POST -H 'Content-type: application/json' \
    #   --data "{\"text\":\"AUCTION VIEWER ALERT: ${message}\"}" \
    #   YOUR_SLACK_WEBHOOK_URL
}

# Check if service is running in PM2
check_pm2_status() {
    if pm2 show "${SERVICE_NAME}" | grep -q "online"; then
        return 0
    else
        return 1
    fi
}

# Check HTTP endpoint with timeout
check_endpoint() {
    local endpoint=$1
    local timeout=5

    START_TIME=$(date +%s%3N)
    RESPONSE=$(curl -s -w "\n%{http_code}\n%{time_total}" --max-time ${timeout} "${endpoint}" 2>/dev/null || echo "000")
    END_TIME=$(date +%s%3N)

    HTTP_CODE=$(echo "${RESPONSE}" | tail -n 2 | head -n 1)
    RESPONSE_TIME=$((END_TIME - START_TIME))

    echo "${HTTP_CODE}:${RESPONSE_TIME}"
}

# Main health check
perform_health_check() {
    local status="HEALTHY"
    local issues=()

    # Check 1: PM2 Process Status
    if ! check_pm2_status; then
        status="CRITICAL"
        issues+=("PM2 process not running")
        log "CRITICAL: ${SERVICE_NAME} not running in PM2"

        # Auto-restart if not running
        log "Attempting to restart ${SERVICE_NAME}..."
        cd /root/Projects/handbag-auth-nextjs/auction-viewer
        pm2 start server.js --name "${SERVICE_NAME}" 2>&1 | tee -a "${LOG_FILE}"
        sleep 5

        if check_pm2_status; then
            log "Service restarted successfully"
            send_alert "Service was down and has been auto-restarted"
        else
            log "CRITICAL: Failed to restart service"
            send_alert "Service down and auto-restart failed - manual intervention required"
            return 1
        fi
    fi

    # Check 2: Health Endpoint
    HEALTH_CHECK=$(check_endpoint "${HEALTH_ENDPOINT}")
    HEALTH_CODE=$(echo "${HEALTH_CHECK}" | cut -d: -f1)
    HEALTH_TIME=$(echo "${HEALTH_CHECK}" | cut -d: -f2)

    if [ "${HEALTH_CODE}" != "200" ]; then
        status="UNHEALTHY"
        issues+=("Health endpoint returned ${HEALTH_CODE}")
        log "WARNING: Health endpoint returned ${HEALTH_CODE}"
    fi

    if [ "${HEALTH_TIME}" -gt "${MAX_RESPONSE_TIME}" ]; then
        status="DEGRADED"
        issues+=("Slow response time: ${HEALTH_TIME}ms")
        log "WARNING: Slow response time: ${HEALTH_TIME}ms"
    fi

    # Check 3: Readiness Endpoint
    READY_CHECK=$(check_endpoint "${READY_ENDPOINT}")
    READY_CODE=$(echo "${READY_CHECK}" | cut -d: -f1)

    if [ "${READY_CODE}" != "200" ]; then
        status="NOT_READY"
        issues+=("Readiness check failed")
        log "WARNING: Readiness check failed"
    fi

    # Check 4: Database Accessibility
    DB_PATH="/root/Projects/handbag-auth-nextjs/data/auction-history/auctions.db"
    if [ ! -f "${DB_PATH}" ]; then
        status="CRITICAL"
        issues+=("Database file not found")
        log "CRITICAL: Database file not found"
        send_alert "Database file missing at ${DB_PATH}"
    else
        # Test database integrity
        if ! sqlite3 "${DB_PATH}" "SELECT COUNT(*) FROM auctions;" > /dev/null 2>&1; then
            status="CRITICAL"
            issues+=("Database query failed")
            log "CRITICAL: Database query failed"
            send_alert "Database query failed - possible corruption"
        fi
    fi

    # Check 5: Disk Space
    DISK_USAGE=$(df -h /root/Projects/handbag-auth-nextjs | tail -1 | awk '{print $5}' | sed 's/%//')
    if [ "${DISK_USAGE}" -gt 90 ]; then
        status="WARNING"
        issues+=("Disk usage at ${DISK_USAGE}%")
        log "WARNING: Disk usage at ${DISK_USAGE}%"
        send_alert "Disk usage critical: ${DISK_USAGE}%"
    fi

    # Check 6: Memory Usage
    MEM_USAGE=$(pm2 show "${SERVICE_NAME}" | grep memory | awk '{print $4}')
    log "Memory usage: ${MEM_USAGE}"

    # Log metrics
    log_metrics "status=${status},health_code=${HEALTH_CODE},response_time=${HEALTH_TIME}ms,disk=${DISK_USAGE}%,memory=${MEM_USAGE}"

    # Report status
    if [ "${status}" == "HEALTHY" ]; then
        log "Health check PASSED - All systems operational (response: ${HEALTH_TIME}ms)"
    else
        log "Health check FAILED - Status: ${status}"
        for issue in "${issues[@]}"; do
            log "  - ${issue}"
        done
    fi

    return 0
}

# Execute health check
perform_health_check

# Cleanup old logs (keep last 1000 lines)
if [ -f "${LOG_FILE}" ]; then
    tail -n 1000 "${LOG_FILE}" > "${LOG_FILE}.tmp"
    mv "${LOG_FILE}.tmp" "${LOG_FILE}"
fi