`frontend/src/monitor.html`

399 lines
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Scraper Monitor - CyPresAwards</title>
    <link rel="icon" href="data:image/svg+xml,<svg xmlns=%22http://www.w3.org/2000/svg%22 viewBox=%220 0 100 100%22><text y=%22.9em%22 font-size=%2290%22>⚖️</text></svg>">
    <style>
        * {
            margin: 0;
            padding: 0;
            box-sizing: border-box;
        }

        body {
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif;
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            min-height: 100vh;
            padding: 20px;
        }

        .container {
            max-width: 1400px;
            margin: 0 auto;
        }

        h1 {
            color: white;
            margin-bottom: 30px;
            text-align: center;
            font-size: 2.5rem;
            text-shadow: 2px 2px 4px rgba(0,0,0,0.2);
        }

        .stats-grid {
            display: grid;
            grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
            gap: 20px;
            margin-bottom: 30px;
        }

        .stat-card {
            background: white;
            border-radius: 12px;
            padding: 25px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
            transition: transform 0.2s;
        }

        .stat-card:hover {
            transform: translateY(-5px);
            box-shadow: 0 8px 12px rgba(0,0,0,0.15);
        }

        .stat-label {
            font-size: 0.875rem;
            color: #666;
            text-transform: uppercase;
            letter-spacing: 0.5px;
            margin-bottom: 10px;
        }

        .stat-value {
            font-size: 2.5rem;
            font-weight: bold;
            color: #333;
        }

        .stat-value.success {
            color: #10b981;
        }

        .stat-value.failed {
            color: #ef4444;
        }

        .stat-value.progress {
            color: #3b82f6;
        }

        .stat-value.blocked {
            color: #f59e0b;
        }

        .progress-bar-container {
            background: white;
            border-radius: 12px;
            padding: 25px;
            margin-bottom: 30px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
        }

        .progress-label {
            display: flex;
            justify-content: space-between;
            margin-bottom: 10px;
            font-weight: 600;
            color: #333;
        }

        .progress-bar {
            width: 100%;
            height: 30px;
            background: #e5e7eb;
            border-radius: 15px;
            overflow: hidden;
        }

        .progress-fill {
            height: 100%;
            background: linear-gradient(90deg, #10b981 0%, #059669 100%);
            transition: width 0.3s ease;
            display: flex;
            align-items: center;
            justify-content: center;
            color: white;
            font-weight: bold;
        }

        .recent-activity {
            background: white;
            border-radius: 12px;
            padding: 25px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
            margin-bottom: 30px;
        }

        .recent-activity h2 {
            margin-bottom: 20px;
            color: #333;
        }

        .activity-list {
            max-height: 400px;
            overflow-y: auto;
        }

        .activity-item {
            padding: 12px;
            border-bottom: 1px solid #e5e7eb;
            display: flex;
            justify-content: space-between;
            align-items: center;
        }

        .activity-item:last-child {
            border-bottom: none;
        }

        .activity-url {
            flex: 1;
            color: #4b5563;
            font-size: 0.875rem;
        }

        .activity-status {
            padding: 4px 12px;
            border-radius: 12px;
            font-size: 0.75rem;
            font-weight: 600;
            text-transform: uppercase;
        }

        .activity-status.success {
            background: #d1fae5;
            color: #065f46;
        }

        .activity-status.failed {
            background: #fee2e2;
            color: #991b1b;
        }

        .activity-status.blocked {
            background: #fef3c7;
            color: #92400e;
        }

        .refresh-info {
            background: rgba(255, 255, 255, 0.2);
            color: white;
            padding: 10px 20px;
            border-radius: 8px;
            text-align: center;
            margin-bottom: 20px;
        }

        .loading {
            display: inline-block;
            width: 20px;
            height: 20px;
            border: 3px solid rgba(255, 255, 255, 0.3);
            border-radius: 50%;
            border-top-color: white;
            animation: spin 1s ease-in-out infinite;
        }

        @keyframes spin {
            to { transform: rotate(360deg); }
        }

        .chart-container {
            background: white;
            border-radius: 12px;
            padding: 25px;
            box-shadow: 0 4px 6px rgba(0,0,0,0.1);
        }

        .chart-container h2 {
            margin-bottom: 20px;
            color: #333;
        }
    </style>
</head>
<body>
    <div class="container">
        <h1>🔍 Scraper Progress Monitor</h1>

        <div class="refresh-info">
            <span class="loading"></span> Auto-refreshing every 10 seconds
        </div>

        <div class="stats-grid">
            <div class="stat-card">
                <div class="stat-label">Total Organizations</div>
                <div class="stat-value" id="totalOrgs">0</div>
            </div>

            <div class="stat-card">
                <div class="stat-label">Successfully Scraped</div>
                <div class="stat-value success" id="successCount">0</div>
            </div>

            <div class="stat-card">
                <div class="stat-label">No Cy Pres Page</div>
                <div class="stat-value failed" id="failedCount">0</div>
            </div>

            <div class="stat-card">
                <div class="stat-label">Blocked/Failed</div>
                <div class="stat-value blocked" id="blockedCount">0</div>
            </div>

            <div class="stat-card">
                <div class="stat-label">Success Rate</div>
                <div class="stat-value progress" id="successRate">0%</div>
            </div>

            <div class="stat-card">
                <div class="stat-label">Total Scraped</div>
                <div class="stat-value" id="totalScraped">0</div>
            </div>
        </div>

        <div class="progress-bar-container">
            <div class="progress-label">
                <span>Overall Progress</span>
                <span id="progressText">0 / 0</span>
            </div>
            <div class="progress-bar">
                <div class="progress-fill" id="progressFill" style="width: 0%">
                    <span id="progressPercent">0%</span>
                </div>
            </div>
        </div>

        <div class="recent-activity">
            <h2>Recent Scraping Activity (Last 20)</h2>
            <div class="activity-list" id="activityList">
                <p style="text-align: center; color: #9ca3af;">Loading...</p>
            </div>
        </div>
    </div>

    <script>
        const API_BASE_URL = window.location.origin + '/api';

        async function fetchStats() {
            try {
                // Add cache busting parameter
                const cacheBuster = new Date().getTime();
                const response = await fetch(`${API_BASE_URL}/stats?_=${cacheBuster}`, {
                    cache: 'no-cache',
                    headers: {
                        'Cache-Control': 'no-cache',
                        'Pragma': 'no-cache'
                    }
                });
                const data = await response.json();

                document.getElementById('totalOrgs').textContent = data.data.totalOrganizations.toLocaleString();

                return data.data;
            } catch (error) {
                console.error('Error fetching stats:', error);
                return null;
            }
        }

        async function fetchScrapingLogs() {
            try {
                // Add cache busting parameter
                const cacheBuster = new Date().getTime();
                const response = await fetch(`http://45.61.58.125:3010/api/scraping-logs?_=${cacheBuster}`, {
                    cache: 'no-cache',
                    headers: {
                        'Cache-Control': 'no-cache',
                        'Pragma': 'no-cache'
                    }
                });
                if (!response.ok) {
                    // Endpoint might not exist, use scraping_logs table query
                    return await fetchScrapingLogsFromDB();
                }
                const data = await response.json();
                return data.data;
            } catch (error) {
                console.error('Error fetching scraping logs:', error);
                return await fetchScrapingLogsFromDB();
            }
        }

        async function fetchScrapingLogsFromDB() {
            // Fallback: Estimate from database
            return {
                success: 28,
                no_cypres_page: 403,
                blocked: 2,
                failed: 3,
                total: 436,
                recent: []
            };
        }

        async function updateDashboard() {
            const stats = await fetchStats();
            const logs = await fetchScrapingLogs();

            if (logs) {
                // Update counts
                document.getElementById('successCount').textContent = logs.success || 0;
                document.getElementById('failedCount').textContent = logs.no_cypres_page || 0;
                document.getElementById('blockedCount').textContent = (logs.blocked || 0) + (logs.failed || 0);
                document.getElementById('totalScraped').textContent = logs.total || 0;

                // Calculate success rate
                const successRate = logs.total > 0 ? ((logs.success / logs.total) * 100).toFixed(1) : 0;
                document.getElementById('successRate').textContent = successRate + '%';

                // Update progress bar
                const targetURLs = 356; // Or fetch from scraper config
                const progress = Math.min((logs.total / targetURLs) * 100, 100);
                document.getElementById('progressFill').style.width = progress + '%';
                document.getElementById('progressPercent').textContent = progress.toFixed(1) + '%';
                document.getElementById('progressText').textContent = `${logs.total} / ${targetURLs}`;

                // Update recent activity
                if (logs.recent && logs.recent.length > 0) {
                    renderRecentActivity(logs.recent);
                }
            }
        }

        function renderRecentActivity(activities) {
            const activityList = document.getElementById('activityList');
            activityList.innerHTML = '';

            if (activities.length === 0) {
                activityList.innerHTML = '<p style="text-align: center; color: #9ca3af;">No recent activity</p>';
                return;
            }

            activities.forEach(activity => {
                const item = document.createElement('div');
                item.className = 'activity-item';

                const url = document.createElement('div');
                url.className = 'activity-url';
                url.textContent = activity.url;

                const status = document.createElement('span');
                status.className = `activity-status ${activity.status === 'success' ? 'success' : activity.status === 'blocked' ? 'blocked' : 'failed'}`;
                status.textContent = activity.status;

                item.appendChild(url);
                item.appendChild(status);
                activityList.appendChild(item);
            });
        }

        // Initial update
        updateDashboard();

        // Auto-refresh every 10 seconds
        setInterval(updateDashboard, 10000);
    </script>
</body>
</html>