`crawler-with-notifications.js`

104 lines
#!/usr/bin/env node
/**
 * Slow Crawler with Steve Notifications
 * Runs continuously, notifies CEO on every successful crawl
 */

import SlowCrawler from './utils/slow-crawler.js';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const STEVE_WEBHOOK = 'http://localhost:9800/api/webhook/message';
const MUSEUM_URL = 'http://45.61.58.125:7600/museum/';
const PROGRESS_FILE = path.join(__dirname, 'museum/crawl-data/progress.json');

let lastNotifiedCount = 0;

async function notifySteve(progress) {
  try {
    const totalResults = progress.results?.length || 0;

    // Only notify if we have new results
    if (totalResults === 0 || totalResults === lastNotifiedCount) {
      return;
    }

    const totalValue = progress.results.reduce((sum, r) => sum + (r.price || 0), 0);

    const message = {
      from: 'Omega Watch Museum Crawler',
      type: 'success',
      priority: 'normal',
      subject: `🏛️ Museum Update: ${totalResults} Omega Watches Found`,
      body: `
**Slow Crawler Progress Update**

✅ **${totalResults} museum-quality pieces** cataloged
💰 **Total value:** $${totalValue.toLocaleString()}
📊 **Pages crawled:** ${progress.currentPage}
🔗 **View museum:** ${MUSEUM_URL}

**Top 5 Most Expensive:**
${progress.results
  .sort((a, b) => b.price - a.price)
  .slice(0, 5)
  .map((w, i) => `${i + 1}. ${w.title.substring(0, 60)}... - $${w.price.toLocaleString()} (${w.source})`)
  .join('\n')}

Updated: ${new Date(progress.lastUpdate).toLocaleString()}
      `.trim(),
      timestamp: new Date().toISOString()
    };

    const response = await fetch(STEVE_WEBHOOK, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(message)
    });

    if (response.ok) {
      console.log(`✅ Notified Steve: ${totalResults} results, $${totalValue.toLocaleString()} total`);
      lastNotifiedCount = totalResults;
    }
  } catch (error) {
    console.error('⚠️ Failed to notify Steve:', error.message);
  }
}

async function startCrawlerWithNotifications() {
  const crawler = new SlowCrawler({
    interval: 60000, // 60 seconds between requests
    maxPages: 500    // Crawl 500 pages
  });

  // Override saveProgress to add notifications
  const originalSaveProgress = crawler.saveProgress.bind(crawler);
  crawler.saveProgress = function() {
    originalSaveProgress();

    // Read the saved progress and notify Steve
    if (fs.existsSync(PROGRESS_FILE)) {
      const progress = JSON.parse(fs.readFileSync(PROGRESS_FILE, 'utf8'));
      notifySteve(progress).catch(console.error);
    }
  };

  console.log('🏛️ Starting Museum Crawler with Steve Notifications');
  console.log(`📨 Will notify Steve at: ${STEVE_WEBHOOK}`);
  console.log(`🌐 Museum URL: ${MUSEUM_URL}\n`);

  try {
    await crawler.startSlowCrawl();
    console.log('\n✅ Crawler completed successfully!');
  } catch (error) {
    console.error('\n❌ Crawler failed:', error);
    process.exit(1);
  }
}

startCrawlerWithNotifications();