`scripts/monitor-bubbe.sh`

182 lines
#!/bin/bash

# Monitor Bubbe.ai Site Health and Auto-Fix
# Runs every 10 minutes via cron to ensure site is working

LOG_FILE="/root/Projects/dear-bubbe-nextjs/monitor.log"
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')

echo "[$TIMESTAMP] ========== Starting Bubbe.ai Health Check ==========" >> "$LOG_FILE"

# Function to log messages
log() {
    echo "[$TIMESTAMP] $1" >> "$LOG_FILE"
}

# Function to rebuild and restart
rebuild_and_restart() {
    log "🔧 Attempting to rebuild and restart..."

    cd /root/Projects/dear-bubbe-nextjs

    # ONLY remove .next if it's genuinely corrupted or missing critical files
    if [ ! -f ".next/BUILD_ID" ] || [ ! -d ".next/server" ]; then
        log "⚠️  .next directory appears corrupted, will rebuild"
        rm -rf .next
        log "✓ Removed corrupted .next directory"
    else
        log "ℹ️  .next directory exists and appears valid, keeping it"
    fi

    # Rebuild
    npm run build >> "$LOG_FILE" 2>&1
    BUILD_EXIT=$?

    if [ $BUILD_EXIT -eq 0 ]; then
        log "✓ Build completed successfully"

        # Restart PM2
        pm2 restart bubbe-nextjs >> "$LOG_FILE" 2>&1
        log "✓ PM2 restarted"

        # Wait for startup
        sleep 5

        return 0
    else
        log "❌ Build failed with exit code $BUILD_EXIT"
        return 1
    fi
}

# Function to restart PM2 only
restart_pm2() {
    log "🔄 Restarting PM2 process..."
    pm2 restart bubbe-nextjs >> "$LOG_FILE" 2>&1

    if [ $? -eq 0 ]; then
        log "✓ PM2 restarted successfully"
        sleep 3
        return 0
    else
        log "❌ PM2 restart failed"
        return 1
    fi
}

# 1. Check if PM2 process is running
log "1️⃣ Checking PM2 process status..."
PM2_STATUS=$(pm2 jlist | jq -r '.[] | select(.name=="bubbe-nextjs") | .pm2_env.status')

if [ "$PM2_STATUS" != "online" ]; then
    log "❌ PM2 process is not online (status: $PM2_STATUS)"
    restart_pm2
    PM2_STATUS=$(pm2 jlist | jq -r '.[] | select(.name=="bubbe-nextjs") | .pm2_env.status')

    if [ "$PM2_STATUS" != "online" ]; then
        log "❌ PM2 restart failed, attempting full rebuild..."
        rebuild_and_restart
    fi
else
    log "✓ PM2 process is online"
fi

# 2. Check if site responds with HTTP 200
log "2️⃣ Checking site HTTP response..."
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://www.bubbe.ai --max-time 10)

if [ "$HTTP_STATUS" != "200" ]; then
    log "❌ Site returned HTTP $HTTP_STATUS (expected 200)"

    # Try restarting PM2 first
    restart_pm2
    sleep 5

    # Check again
    HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://www.bubbe.ai --max-time 10)

    if [ "$HTTP_STATUS" != "200" ]; then
        log "❌ Site still not responding correctly, attempting full rebuild..."
        rebuild_and_restart
        sleep 5

        # Final check
        HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" https://www.bubbe.ai --max-time 10)

        if [ "$HTTP_STATUS" = "200" ]; then
            log "✅ Site restored after rebuild!"
        else
            log "🚨 CRITICAL: Site still not responding after rebuild (HTTP $HTTP_STATUS)"
        fi
    else
        log "✅ Site restored after PM2 restart"
    fi
else
    log "✓ Site responding with HTTP 200"
fi

# 3. Test Voice API endpoint
log "3️⃣ Testing voice API endpoint..."
VOICE_TEST=$(curl -s -X POST http://45.61.58.125:3011/api/bubbe-voice \
    -H "Content-Type: application/json" \
    -d '{"text":"Test","mode":"bubbe"}' \
    -w "\n%{http_code}" \
    --max-time 15 2>&1)

VOICE_HTTP_STATUS=$(echo "$VOICE_TEST" | tail -1)
VOICE_SIZE=$(echo "$VOICE_TEST" | grep -oP 'Content-Length: \K\d+' | head -1)

if [ "$VOICE_HTTP_STATUS" = "200" ] && [ -n "$VOICE_SIZE" ] && [ "$VOICE_SIZE" -gt 1000 ]; then
    log "✓ Voice API working (HTTP 200, ${VOICE_SIZE} bytes)"
elif [ "$VOICE_HTTP_STATUS" = "200" ]; then
    log "⚠️  Voice API returned 200 but audio size may be small (${VOICE_SIZE:-0} bytes)"
else
    log "❌ Voice API failed (HTTP $VOICE_HTTP_STATUS)"

    # Try restarting
    restart_pm2
    sleep 5

    # Test again
    VOICE_TEST_2=$(curl -s -X POST http://45.61.58.125:3011/api/bubbe-voice \
        -H "Content-Type: application/json" \
        -d '{"text":"Test","mode":"bubbe"}' \
        -w "\n%{http_code}" \
        --max-time 15 2>&1)

    VOICE_HTTP_STATUS_2=$(echo "$VOICE_TEST_2" | tail -1)

    if [ "$VOICE_HTTP_STATUS_2" = "200" ]; then
        log "✅ Voice API restored after restart"
    else
        log "🚨 Voice API still failing (HTTP $VOICE_HTTP_STATUS_2)"
    fi
fi

# 4. Check PM2 logs for errors
log "4️⃣ Checking recent PM2 errors..."
ERROR_COUNT=$(pm2 logs bubbe-nextjs --lines 50 --nostream --err 2>/dev/null | grep -i "error" | wc -l)

if [ "$ERROR_COUNT" -gt 5 ]; then
    log "⚠️  Found $ERROR_COUNT errors in recent logs"
    pm2 logs bubbe-nextjs --lines 5 --nostream --err >> "$LOG_FILE" 2>&1
else
    log "✓ No significant errors in recent logs"
fi

# 5. Check disk space
log "5️⃣ Checking disk space..."
DISK_USAGE=$(df -h / | tail -1 | awk '{print $5}' | sed 's/%//')

if [ "$DISK_USAGE" -gt 90 ]; then
    log "⚠️  Disk usage is high: ${DISK_USAGE}%"
else
    log "✓ Disk usage OK: ${DISK_USAGE}%"
fi

# Summary
log "========== Health Check Complete =========="
echo "" >> "$LOG_FILE"

exit 0