`scripts/automated-monitor.sh`

65 lines
#!/bin/bash

# Automated Monitoring Script for Dear Bubbe
# Runs comprehensive checks and restarts services if needed
# Location: /root/Projects/dear-bubbe-nextjs/scripts/automated-monitor.sh

SCRIPT_DIR="/root/Projects/dear-bubbe-nextjs/scripts"
PROJECT_DIR="/root/Projects/dear-bubbe-nextjs"
TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
ALERT_LOG="/root/Projects/dear-bubbe-nextjs/automated-alerts.log"

echo "[$TIMESTAMP] Starting automated monitoring check..." >> $ALERT_LOG

# Function to restart service if needed
restart_service() {
    local reason="$1"
    echo "[$TIMESTAMP] RESTARTING SERVICE: $reason" >> $ALERT_LOG
    
    cd $PROJECT_DIR
    
    # Kill any hung processes on port 3011
    lsof -ti:3011 | xargs kill -9 2>/dev/null || true
    
    # Restart PM2 process
    pm2 restart bubbe-ai
    
    # Wait for startup
    sleep 10
    
    # Verify restart successful
    if pm2 list | grep -q "bubbe-ai.*online"; then
        echo "[$TIMESTAMP] ✅ Service restart successful" >> $ALERT_LOG
    else
        echo "[$TIMESTAMP] ❌ Service restart FAILED" >> $ALERT_LOG
        # Send alert (could integrate with Slack, email, etc.)
    fi
}

# Run health check
echo "[$TIMESTAMP] Running health check..." >> $ALERT_LOG
$SCRIPT_DIR/health-check.sh > /tmp/health_check_results.txt 2>&1
HEALTH_EXIT_CODE=$?

# Run voice monitoring
echo "[$TIMESTAMP] Running voice monitoring..." >> $ALERT_LOG
$SCRIPT_DIR/voice-monitor.sh
VOICE_EXIT_CODE=$?

# Analyze results and take action
if [ $HEALTH_EXIT_CODE -eq 2 ]; then
    restart_service "Critical health check failures detected"
elif [ $VOICE_EXIT_CODE -eq 2 ]; then
    restart_service "Voice features failing (success rate < 80%)"
elif [ $HEALTH_EXIT_CODE -eq 1 ] && [ $VOICE_EXIT_CODE -eq 1 ]; then
    restart_service "Multiple minor issues detected"
else
    echo "[$TIMESTAMP] System status: OK (health: $HEALTH_EXIT_CODE, voice: $VOICE_EXIT_CODE)" >> $ALERT_LOG
fi

# Clean up old log files (keep last 7 days)
find $PROJECT_DIR -name "*.log" -mtime +7 -delete 2>/dev/null || true
find /tmp -name "bubbe_*" -mtime +1 -delete 2>/dev/null || true

echo "[$TIMESTAMP] Automated monitoring check completed" >> $ALERT_LOG
echo "" >> $ALERT_LOG