← back to Watches

utils/slow-crawler.js

401 lines

/**
 * Slow Distributed Crawler for Real Omega Auction Data
 * - Crawls every 60 seconds (1 request per minute)
 * - Rotates user agents
 * - Uses proxies (optional)
 * - Respects robots.txt
 * - Stores results incrementally
 */

import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

class SlowCrawler {
  constructor(options = {}) {
    this.browser = null;
    this.crawlInterval = options.interval || 60000; // 60 seconds default
    this.maxPages = options.maxPages || 500; // Max pages to crawl
    this.currentPage = 0;
    this.results = [];
    this.crawlQueue = [];

    this.outputDir = path.join(__dirname, '../museum/crawl-data');
    if (!fs.existsSync(this.outputDir)) {
      fs.mkdirSync(this.outputDir, { recursive: true });
    }

    // Load existing progress
    this.progressFile = path.join(this.outputDir, 'progress.json');
    this.loadProgress();

    // User agent rotation
    this.userAgents = [
      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
      'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15'
    ];

    // Free proxy rotation (optional - add your own)
    this.proxies = options.proxies || null; // Format: ['http://ip:port', ...]
  }

  loadProgress() {
    if (fs.existsSync(this.progressFile)) {
      const data = JSON.parse(fs.readFileSync(this.progressFile, 'utf8'));
      this.currentPage = data.currentPage || 0;
      this.results = data.results || [];
      console.log(`📂 Loaded progress: ${this.results.length} results, page ${this.currentPage}`);
    }
  }

  saveProgress() {
    const data = {
      currentPage: this.currentPage,
      results: this.results,
      lastUpdate: new Date().toISOString()
    };
    fs.writeFileSync(this.progressFile, JSON.stringify(data, null, 2));
  }

  getRandomUserAgent() {
    return this.userAgents[Math.floor(Math.random() * this.userAgents.length)];
  }

  async initBrowser(proxy = null) {
    const browserOptions = {
      headless: true,
      args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-blink-features=AutomationControlled'
      ]
    };

    if (proxy) {
      browserOptions.proxy = { server: proxy };
    }

    this.browser = await chromium.launch(browserOptions);
  }

  async crawlSothebys(pageNum = 0) {
    if (!this.browser) {
      await this.initBrowser(this.proxies ? this.proxies[pageNum % this.proxies.length] : null);
    }

    const context = await this.browser.newContext({
      userAgent: this.getRandomUserAgent(),
      viewport: { width: 1920, height: 1080 },
      locale: 'en-US',
      timezoneId: 'America/New_York'
    });

    const page = await context.newPage();

    try {
      const from = pageNum * 24; // 24 results per page
      const searchUrl = `https://www.sothebys.com/en/results?from=${from}&size=24&query=omega%20watch`;

      console.log(`\n🔍 [${new Date().toLocaleTimeString()}] Crawling Sotheby's page ${pageNum + 1}...`);
      console.log(`   URL: ${searchUrl}`);

      // Slow navigation to appear human
      await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });

      // Random delay to appear human (2-5 seconds)
      await page.waitForTimeout(2000 + Math.random() * 3000);

      // Try to find results
      const hasResults = await page.locator('.lot-card, [data-testid="lot-card"], .auction-lot').count() > 0;

      if (!hasResults) {
        console.log('   ⚠️  No results found on this page');
        return [];
      }

      // Extract data
      const results = await page.evaluate(() => {
        const items = [];

        // Try multiple selectors
        const selectors = ['.lot-card', '[data-testid="lot-card"]', '.auction-lot', '.result-item'];
        let cards = [];

        for (const selector of selectors) {
          cards = document.querySelectorAll(selector);
          if (cards.length > 0) break;
        }

        cards.forEach(card => {
          try {
            // Extract title
            const titleEl = card.querySelector('h3, h2, .lot-title, [class*="title"]');
            const title = titleEl?.textContent?.trim();

            if (!title || !title.toLowerCase().includes('omega')) return;

            // Extract price
            const priceEl = card.querySelector('.estimate, .price, [class*="price"], [class*="estimate"]');
            const priceText = priceEl?.textContent?.trim() || '';
            const priceMatch = priceText.match(/[\d,]+/g);
            const price = priceMatch ? parseInt(priceMatch[priceMatch.length - 1].replace(/,/g, '')) : null;

            // Extract image
            const imageEl = card.querySelector('img');
            const image = imageEl?.src || imageEl?.getAttribute('data-src');

            // Extract link
            const linkEl = card.querySelector('a');
            const url = linkEl?.href;

            // Extract lot number
            const lotEl = card.querySelector('.lot-number, [class*="lot"]');
            const lotNumber = lotEl?.textContent?.trim();

            // Extract date
            const dateEl = card.querySelector('.date, [class*="date"]');
            const date = dateEl?.textContent?.trim();

            if (title && price && price > 1000) {
              items.push({
                title,
                price,
                image,
                url,
                lotNumber,
                date,
                source: 'Sotheby\'s',
                crawledAt: new Date().toISOString()
              });
            }
          } catch (e) {
            console.error('Error parsing card:', e);
          }
        });

        return items;
      });

      console.log(`   ✓ Found ${results.length} Omega watches`);

      if (results.length > 0) {
        this.results.push(...results);
        this.saveProgress();

        // Log sample
        const sample = results[0];
        console.log(`   Sample: ${sample.title.substring(0, 50)}... - $${sample.price.toLocaleString()}`);
      }

      await page.close();
      await context.close();

      return results;

    } catch (error) {
      console.error(`   ✗ Error: ${error.message}`);
      await page.close();
      await context.close();
      return [];
    }
  }

  async crawlChristies(pageNum = 0) {
    if (!this.browser) {
      await this.initBrowser(this.proxies ? this.proxies[pageNum % this.proxies.length] : null);
    }

    const context = await this.browser.newContext({
      userAgent: this.getRandomUserAgent(),
      viewport: { width: 1920, height: 1080 }
    });

    const page = await context.newPage();

    try {
      const searchUrl = `https://www.christies.com/en/results?keyword=omega%20watch&page=${pageNum + 1}`;

      console.log(`\n🔍 [${new Date().toLocaleTimeString()}] Crawling Christie's page ${pageNum + 1}...`);

      await page.goto(searchUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
      await page.waitForTimeout(2000 + Math.random() * 3000);

      const results = await page.evaluate(() => {
        const items = [];
        const cards = document.querySelectorAll('[data-testid="lot-card"], .lot-card, .auction-lot');

        cards.forEach(card => {
          try {
            const titleEl = card.querySelector('h3, .lot-title');
            const title = titleEl?.textContent?.trim();

            if (!title || !title.toLowerCase().includes('omega')) return;

            const priceEl = card.querySelector('[class*="price"], [class*="estimate"]');
            const priceText = priceEl?.textContent?.trim() || '';
            const priceMatch = priceText.match(/[\d,]+/g);
            const price = priceMatch ? parseInt(priceMatch[priceMatch.length - 1].replace(/,/g, '')) : null;

            const imageEl = card.querySelector('img');
            const image = imageEl?.src;

            const linkEl = card.querySelector('a');
            const url = linkEl?.href;

            if (title && price && price > 1000) {
              items.push({
                title,
                price,
                image,
                url,
                source: 'Christie\'s',
                crawledAt: new Date().toISOString()
              });
            }
          } catch (e) {}
        });

        return items;
      });

      console.log(`   ✓ Found ${results.length} Omega watches`);

      if (results.length > 0) {
        this.results.push(...results);
        this.saveProgress();
      }

      await page.close();
      await context.close();

      return results;

    } catch (error) {
      console.error(`   ✗ Error: ${error.message}`);
      await page.close();
      await context.close();
      return [];
    }
  }

  async startSlowCrawl() {
    console.log('🐌 SLOW CRAWLER STARTED');
    console.log(`⏱️  Interval: ${this.crawlInterval / 1000} seconds per request`);
    console.log(`🎯 Target: ${this.maxPages} pages`);
    console.log(`📊 Current progress: ${this.results.length} results, page ${this.currentPage}\n`);

    const startTime = Date.now();

    while (this.currentPage < this.maxPages) {
      const requestStart = Date.now();

      // Alternate between sources
      const useSothebys = this.currentPage % 2 === 0;

      try {
        if (useSothebys) {
          await this.crawlSothebys(Math.floor(this.currentPage / 2));
        } else {
          await this.crawlChristies(Math.floor(this.currentPage / 2));
        }

        this.currentPage++;
        this.saveProgress();

      } catch (error) {
        console.error(`\n❌ Fatal error on page ${this.currentPage}:`, error.message);
      }

      // Calculate delay to maintain exact interval
      const elapsed = Date.now() - requestStart;
      const delay = Math.max(0, this.crawlInterval - elapsed);

      if (this.currentPage < this.maxPages) {
        console.log(`\n⏸️  Waiting ${Math.round(delay / 1000)}s before next request...`);
        console.log(`📈 Progress: ${this.currentPage}/${this.maxPages} pages (${((this.currentPage / this.maxPages) * 100).toFixed(1)}%)`);
        console.log(`💾 Total results: ${this.results.length}`);

        await new Promise(resolve => setTimeout(resolve, delay));
      }

      // Close browser every 10 requests to avoid detection
      if (this.currentPage % 10 === 0 && this.browser) {
        await this.browser.close();
        this.browser = null;
        console.log('🔄 Browser restarted for stealth');
      }
    }

    const totalTime = (Date.now() - startTime) / 1000 / 60;

    console.log('\n✅ CRAWL COMPLETE');
    console.log(`📊 Total results: ${this.results.length}`);
    console.log(`⏱️  Total time: ${totalTime.toFixed(1)} minutes`);
    console.log(`💾 Data saved: ${this.progressFile}`);

    // Generate final output
    this.generateFinalOutput();

    if (this.browser) {
      await this.browser.close();
    }
  }

  generateFinalOutput() {
    // Sort by price
    const sorted = this.results
      .filter(r => r.price > 0)
      .sort((a, b) => b.price - a.price);

    // Remove duplicates
    const unique = [];
    const seen = new Set();

    for (const item of sorted) {
      const key = `${item.title}-${item.price}`;
      if (!seen.has(key)) {
        seen.add(key);
        unique.push(item);
      }
    }

    const output = {
      crawlComplete: new Date().toISOString(),
      totalResults: unique.length,
      totalValue: unique.reduce((sum, r) => sum + r.price, 0),
      sources: ['Sotheby\'s', 'Christie\'s'],
      results: unique.slice(0, 100) // Top 100
    };

    const outputPath = path.join(this.outputDir, 'omega-auction-results.json');
    fs.writeFileSync(outputPath, JSON.stringify(output, null, 2));

    console.log(`\n📄 Final output: ${outputPath}`);
  }
}

export default SlowCrawler;

// CLI usage
if (import.meta.url === `file://${process.argv[1]}`) {
  const args = process.argv.slice(2);
  const interval = args[0] ? parseInt(args[0]) * 1000 : 60000; // Default 60 seconds
  const maxPages = args[1] ? parseInt(args[1]) : 100;

  const crawler = new SlowCrawler({
    interval,
    maxPages,
    // Add proxies here if you have them:
    // proxies: ['http://proxy1:port', 'http://proxy2:port']
  });

  crawler.startSlowCrawl().catch(console.error);
}