`scripts/health-check.sh`

138 lines
#!/bin/bash

# Health check script for Dear Bubbe - Runs every 5 minutes to ensure 24/7 uptime
# Usage: ./scripts/health-check.sh

LOG_FILE="/root/Projects/dear-bubbe-nextjs/health-monitor.log"
DATE_TIME=$(date '+%Y-%m-%d %H:%M:%S')
SERVER_URL="http://45.61.58.125:3011"
VOICE_API_URL="http://45.61.58.125:3011/api/bubbe-voice"

# Color codes for output
RED='\033[0;31m'
GREEN='\033[0;32m' 
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

echo "[$DATE_TIME] ========== DEAR BUBBE HEALTH CHECK ==========" | tee -a "$LOG_FILE"

# Test 1: Check if PM2 process is running
echo -n "[$DATE_TIME] Checking PM2 process status... " | tee -a "$LOG_FILE"
PM2_STATUS=$(pm2 jlist | jq '.[] | select(.name=="bubbe") | .pm2_env.status')
if [[ "$PM2_STATUS" == '"online"' ]]; then
    echo -e "${GREEN}✅ PASS${NC} - PM2 process online" | tee -a "$LOG_FILE"
    PM2_PASS=1
else
    echo -e "${RED}❌ FAIL${NC} - PM2 process not online (Status: $PM2_STATUS)" | tee -a "$LOG_FILE"
    PM2_PASS=0
fi

# Test 2: Check if port 3011 is responding
echo -n "[$DATE_TIME] Checking port 3011 response... " | tee -a "$LOG_FILE"
if curl -s --max-time 10 "$SERVER_URL" > /dev/null; then
    echo -e "${GREEN}✅ PASS${NC} - App responding on port 3011" | tee -a "$LOG_FILE"
    PORT_PASS=1
else
    echo -e "${RED}❌ FAIL${NC} - App not responding on port 3011" | tee -a "$LOG_FILE"
    PORT_PASS=0
fi

# Test 3: Check if /api/chat endpoint works
echo -n "[$DATE_TIME] Testing chat API endpoint... " | tee -a "$LOG_FILE"
CHAT_RESPONSE=$(curl -s --max-time 15 -X POST "$SERVER_URL/api/chat" \
    -H "Content-Type: application/json" \
    -d '{"message":"health check","userId":"health_test","mode":"bubbe"}' \
    2>/dev/null)

if echo "$CHAT_RESPONSE" | grep -q "response"; then
    echo -e "${GREEN}✅ PASS${NC} - Chat API responding" | tee -a "$LOG_FILE"
    API_PASS=1
else
    echo -e "${RED}❌ FAIL${NC} - Chat API not responding properly" | tee -a "$LOG_FILE"
    echo "[$DATE_TIME] Chat API Response: $CHAT_RESPONSE" >> "$LOG_FILE"
    API_PASS=0
fi

# Test 4: Check ElevenLabs voice API
echo -n "[$DATE_TIME] Testing ElevenLabs voice API... " | tee -a "$LOG_FILE"
VOICE_RESPONSE=$(curl -s --max-time 20 -X POST "$VOICE_API_URL" \
    -H "Content-Type: application/json" \
    -d '{"text":"Health check test","voiceId":"21m00Tcm4TlvDq8ikWAM"}' \
    2>/dev/null)

if [[ ${#VOICE_RESPONSE} -gt 100 ]]; then
    echo -e "${GREEN}✅ PASS${NC} - Voice API responding with audio data" | tee -a "$LOG_FILE"
    VOICE_PASS=1
else
    echo -e "${YELLOW}⚠️ WARN${NC} - Voice API may have issues (Response size: ${#VOICE_RESPONSE})" | tee -a "$LOG_FILE"
    VOICE_PASS=0
fi

# Test 5: Memory usage check
echo -n "[$DATE_TIME] Checking memory usage... " | tee -a "$LOG_FILE"
MEMORY_USAGE=$(pm2 jlist | jq '.[] | select(.name=="bubbe") | .pm2_env.axm_monitor.Memory.value' | sed 's/"//g' | sed 's/ MB//')
if [[ "$MEMORY_USAGE" -lt 500 ]]; then
    echo -e "${GREEN}✅ PASS${NC} - Memory usage: ${MEMORY_USAGE}MB" | tee -a "$LOG_FILE"
    MEMORY_PASS=1
else
    echo -e "${YELLOW}⚠️ WARN${NC} - High memory usage: ${MEMORY_USAGE}MB" | tee -a "$LOG_FILE"
    MEMORY_PASS=1
fi

# Calculate overall health score
TOTAL_TESTS=5
PASSED_TESTS=$((PM2_PASS + PORT_PASS + API_PASS + VOICE_PASS + MEMORY_PASS))
HEALTH_SCORE=$((PASSED_TESTS * 100 / TOTAL_TESTS))

echo "[$DATE_TIME] Health Score: $HEALTH_SCORE% ($PASSED_TESTS/$TOTAL_TESTS tests passed)" | tee -a "$LOG_FILE"

# Auto-restart logic if critical tests fail
if [[ $PM2_PASS -eq 0 ]] || [[ $PORT_PASS -eq 0 ]]; then
    echo -e "[$DATE_TIME] ${RED}🚨 CRITICAL FAILURE DETECTED${NC} - Attempting auto-restart..." | tee -a "$LOG_FILE"
    
    # Kill any processes using port 3011
    PIDS=$(lsof -ti:3011 2>/dev/null)
    if [[ -n "$PIDS" ]]; then
        echo "[$DATE_TIME] Killing processes on port 3011: $PIDS" | tee -a "$LOG_FILE"
        echo "$PIDS" | xargs kill -9 2>/dev/null
        sleep 3
    fi
    
    # Restart PM2 process
    echo "[$DATE_TIME] Restarting PM2 bubbe process..." | tee -a "$LOG_FILE"
    cd /root/Projects/dear-bubbe-nextjs
    pm2 restart bubbe 2>&1 | tee -a "$LOG_FILE"
    
    # Wait and retest
    echo "[$DATE_TIME] Waiting 10 seconds for restart to complete..." | tee -a "$LOG_FILE"
    sleep 10
    
    # Quick retest
    if curl -s --max-time 10 "$SERVER_URL" > /dev/null; then
        echo -e "[$DATE_TIME] ${GREEN}✅ RESTART SUCCESSFUL${NC} - App responding again" | tee -a "$LOG_FILE"
        RESTART_SUCCESS=1
    else
        echo -e "[$DATE_TIME] ${RED}❌ RESTART FAILED${NC} - App still not responding" | tee -a "$LOG_FILE"
        RESTART_SUCCESS=0
        
        # Try building and restarting if simple restart failed
        echo "[$DATE_TIME] Attempting full rebuild and restart..." | tee -a "$LOG_FILE"
        npm run build 2>&1 | tee -a "$LOG_FILE"
        pm2 restart bubbe 2>&1 | tee -a "$LOG_FILE"
        sleep 10
        
        if curl -s --max-time 10 "$SERVER_URL" > /dev/null; then
            echo -e "[$DATE_TIME] ${GREEN}✅ REBUILD SUCCESSFUL${NC} - App responding after rebuild" | tee -a "$LOG_FILE"
        else
            echo -e "[$DATE_TIME] ${RED}❌ REBUILD FAILED${NC} - Manual intervention required" | tee -a "$LOG_FILE"
        fi
    fi
else
    echo -e "[$DATE_TIME] ${GREEN}✅ ALL SYSTEMS OPERATIONAL${NC}" | tee -a "$LOG_FILE"
fi

# Keep only last 1000 lines of log file
tail -n 1000 "$LOG_FILE" > "$LOG_FILE.tmp" && mv "$LOG_FILE.tmp" "$LOG_FILE"

echo "[$DATE_TIME] =============================================" | tee -a "$LOG_FILE"
echo "" | tee -a "$LOG_FILE"