`scripts/devops/validate-devops.sh`

187 lines
#!/bin/bash
# DevOps Infrastructure Validation Script
# Tests all DevOps components to ensure proper functionality

set -euo pipefail

GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
NC='\033[0m'

PASSED=0
FAILED=0

log_success() {
    echo -e "${GREEN}✓${NC} $1"
    ((PASSED++))
}

log_failure() {
    echo -e "${RED}✗${NC} $1"
    ((FAILED++))
}

log_info() {
    echo -e "${YELLOW}ℹ${NC} $1"
}

echo "=========================================="
echo "LUXVAULT DevOps Validation"
echo "=========================================="
echo ""

# Test 1: Service Running
log_info "Test 1: Checking if service is running..."
if pm2 show auction-viewer | grep -q "online"; then
    log_success "Service is running"
else
    log_failure "Service is not running"
fi

# Test 2: Health Endpoint
log_info "Test 2: Testing health endpoint..."
HEALTH_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" http://45.61.58.125:7500/health)
if [ "$HEALTH_RESPONSE" == "200" ]; then
    log_success "Health endpoint responding (HTTP 200)"
else
    log_failure "Health endpoint failed (HTTP $HEALTH_RESPONSE)"
fi

# Test 3: Readiness Endpoint
log_info "Test 3: Testing readiness endpoint..."
READY_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" http://45.61.58.125:7500/ready)
if [ "$READY_RESPONSE" == "200" ]; then
    log_success "Readiness endpoint responding (HTTP 200)"
else
    log_failure "Readiness endpoint failed (HTTP $READY_RESPONSE)"
fi

# Test 4: Metrics Endpoint
log_info "Test 4: Testing metrics endpoint..."
METRICS_RESPONSE=$(curl -s http://45.61.58.125:7500/metrics)
if echo "$METRICS_RESPONSE" | grep -q "auction_viewer_requests_total"; then
    log_success "Metrics endpoint working"
else
    log_failure "Metrics endpoint not returning expected data"
fi

# Test 5: Database Connectivity
log_info "Test 5: Testing database connectivity..."
DB_PATH="/root/Projects/handbag-auth-nextjs/data/auction-history/auctions.db"
if [ -f "$DB_PATH" ]; then
    if sqlite3 "$DB_PATH" "SELECT COUNT(*) FROM auctions;" > /dev/null 2>&1; then
        log_success "Database accessible"
    else
        log_failure "Database query failed"
    fi
else
    log_failure "Database file not found"
fi

# Test 6: Backup Script Exists
log_info "Test 6: Checking backup script..."
BACKUP_SCRIPT="/root/Projects/handbag-auth-nextjs/scripts/devops/backup-database.sh"
if [ -x "$BACKUP_SCRIPT" ]; then
    log_success "Backup script exists and is executable"
else
    log_failure "Backup script missing or not executable"
fi

# Test 7: Health Check Script
log_info "Test 7: Checking health check script..."
HEALTH_SCRIPT="/root/Projects/handbag-auth-nextjs/scripts/devops/health-check.sh"
if [ -x "$HEALTH_SCRIPT" ]; then
    log_success "Health check script exists and is executable"
else
    log_failure "Health check script missing or not executable"
fi

# Test 8: Deployment Script
log_info "Test 8: Checking deployment script..."
DEPLOY_SCRIPT="/root/Projects/handbag-auth-nextjs/scripts/devops/deploy.sh"
if [ -x "$DEPLOY_SCRIPT" ]; then
    log_success "Deployment script exists and is executable"
else
    log_failure "Deployment script missing or not executable"
fi

# Test 9: Backup Directory
log_info "Test 9: Checking backup directory..."
BACKUP_DIR="/root/Projects/handbag-auth-nextjs/backups"
if [ -d "$BACKUP_DIR" ]; then
    BACKUP_COUNT=$(find "$BACKUP_DIR" -name "*.tar.gz" -type f | wc -l)
    log_success "Backup directory exists ($BACKUP_COUNT backups found)"
else
    log_failure "Backup directory not found"
fi

# Test 10: Cron Jobs
log_info "Test 10: Checking cron jobs..."
if crontab -l | grep -q "LUXVAULT Auction Viewer"; then
    log_success "DevOps cron jobs configured"
else
    log_failure "DevOps cron jobs not found in crontab"
fi

# Test 11: Log Directory
log_info "Test 11: Checking log directory..."
LOG_DIR="/root/Projects/handbag-auth-nextjs/logs"
if [ -d "$LOG_DIR" ]; then
    log_success "Log directory exists"
else
    log_failure "Log directory not found"
fi

# Test 12: PM2 Saved
log_info "Test 12: Checking PM2 configuration..."
if [ -f ~/.pm2/dump.pm2 ]; then
    log_success "PM2 configuration saved"
else
    log_failure "PM2 configuration not saved"
fi

# Test 13: API Endpoints
log_info "Test 13: Testing API endpoints..."
API_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" http://45.61.58.125:7500/api/stats)
if [ "$API_RESPONSE" == "200" ]; then
    log_success "API endpoints working"
else
    log_failure "API endpoints failed (HTTP $API_RESPONSE)"
fi

# Test 14: Documentation
log_info "Test 14: Checking documentation..."
if [ -f "/root/Projects/handbag-auth-nextjs/DEVOPS_RUNBOOK.md" ]; then
    log_success "DevOps runbook exists"
else
    log_failure "DevOps runbook not found"
fi

# Test 15: Monitoring Dashboard
log_info "Test 15: Checking monitoring dashboard..."
DASHBOARD="/root/Projects/handbag-auth-nextjs/scripts/devops/monitoring-dashboard.html"
if [ -f "$DASHBOARD" ]; then
    log_success "Monitoring dashboard exists"
else
    log_failure "Monitoring dashboard not found"
fi

echo ""
echo "=========================================="
echo "Validation Summary"
echo "=========================================="
echo -e "${GREEN}Passed:${NC} $PASSED"
echo -e "${RED}Failed:${NC} $FAILED"
echo "Total Tests: $((PASSED + FAILED))"

if [ $FAILED -eq 0 ]; then
    echo ""
    echo -e "${GREEN}✓ All tests passed! DevOps infrastructure is operational.${NC}"
    exit 0
else
    echo ""
    echo -e "${RED}✗ Some tests failed. Please review and fix issues.${NC}"
    exit 1
fi