`scripts/devops/health-monitor-simple.sh`

112 lines
#!/bin/bash
#
# LUXVAULT Simple Health Monitor
# Monitors application health and sends alerts on failures
#

PROJECT_DIR="/root/Projects/handbag-auth-nextjs"
LOG_FILE="$PROJECT_DIR/logs/health-monitor.log"
ALERT_LOG="$PROJECT_DIR/logs/alerts.log"

# Create log directories
mkdir -p "$PROJECT_DIR/logs"

log() {
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"
}

alert() {
    local severity="$1"
    local message="$2"
    echo "[$(date +'%Y-%m-%d %H:%M:%S')] [$severity] $message" | tee -a "$ALERT_LOG"
}

log "========================================="
log "Health Check Started"

# Check 1: PM2 Status
log "Checking PM2 status..."
if pm2 status auction-viewer 2>/dev/null | grep -q "online"; then
    log "✓ PM2: auction-viewer is online"
else
    alert "CRITICAL" "PM2: auction-viewer is NOT running!"
    log "Attempting auto-restart..."
    pm2 restart auction-viewer 2>&1 | tee -a "$LOG_FILE"
    sleep 3
fi

# Check 2: HTTP Endpoint
log "Checking HTTP endpoint..."
if curl -sf --max-time 5 "http://45.61.58.125:7500/api/status" > /dev/null 2>&1; then
    log "✓ HTTP: Service responding"
else
    alert "CRITICAL" "HTTP: Service not responding!"
fi

# Check 3: API Functionality
log "Checking API functionality..."
if curl -sf --max-time 5 "http://45.61.58.125:7500/api/auctions?limit=1" > /dev/null 2>&1; then
    log "✓ API: Endpoint working"
else
    alert "ERROR" "API: Endpoint not working"
fi

# Check 4: Database
log "Checking database..."
DB_PATH="$PROJECT_DIR/data/auction-history/auctions.db"
if [ -f "$DB_PATH" ]; then
    DB_SIZE=$(du -h "$DB_PATH" | cut -f1)
    RECORD_COUNT=$(sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM auctions;" 2>/dev/null || echo "ERROR")

    if [ "$RECORD_COUNT" != "ERROR" ]; then
        log "✓ Database: $RECORD_COUNT records, size: $DB_SIZE"
    else
        alert "ERROR" "Database: Unable to query records"
    fi
else
    alert "CRITICAL" "Database: File not found!"
fi

# Check 5: Disk Space
log "Checking disk space..."
DISK_USAGE=$(df /root | tail -1 | awk '{print $5}' | sed 's/%//')
if [ "$DISK_USAGE" -gt 90 ]; then
    alert "CRITICAL" "Disk: Usage at ${DISK_USAGE}%"
elif [ "$DISK_USAGE" -gt 80 ]; then
    alert "WARNING" "Disk: Usage at ${DISK_USAGE}%"
else
    log "✓ Disk: Usage at ${DISK_USAGE}%"
fi

# Check 6: Scraper Log
log "Checking scraper status..."
SCRAPER_LOG="$PROJECT_DIR/data/auction-history/scraper.log"
if [ -f "$SCRAPER_LOG" ]; then
    LAST_SCRAPE=$(tail -100 "$SCRAPER_LOG" | grep "DAILY SCRAPE COMPLETED" | tail -1 || echo "")
    if [ -n "$LAST_SCRAPE" ]; then
        log "✓ Scraper: Last completion found"
    else
        HOUR=$(date +%H)
        if [ "$HOUR" -gt 7 ]; then
            alert "WARNING" "Scraper: No completion today (time: ${HOUR}:00)"
        else
            log "✓ Scraper: Before scheduled time"
        fi
    fi
fi

# Check 7: Error Rate
log "Checking error rate..."
ERROR_COUNT=$(pm2 logs auction-viewer --lines 100 --nostream 2>&1 | grep -ci "error" || true)
if [ -z "$ERROR_COUNT" ]; then
    ERROR_COUNT=0
fi
if [ "$ERROR_COUNT" -gt 10 ]; then
    alert "WARNING" "Errors: $ERROR_COUNT errors in last 100 log lines"
else
    log "✓ Errors: $ERROR_COUNT in last 100 lines"
fi

log "Health Check Complete"
log "========================================="