`services/chrono24-scraper.js`

300 lines
/**
 * Chrono24 Public Listing Scraper
 * Scrapes publicly visible listings from Chrono24 search results
 * Respects robots.txt and rate limits
 */

import { setTimeout } from 'timers/promises';

const CHRONO24_BASE = 'https://www.chrono24.com';

// Strict rate limiting: 1 request every 5 seconds
const RATE_LIMIT_MS = 5000;
let lastRequestTime = 0;

// Cache configuration
const cache = new Map();
const CACHE_TTL = 6 * 60 * 60 * 1000; // 6 hours

// Robots.txt compliance - these paths are typically allowed
const ALLOWED_PATHS = [
  '/omega/',
  '/rolex/',
  '/search/'
];

/**
 * Check if path is allowed by robots.txt
 */
function isPathAllowed(path) {
  return ALLOWED_PATHS.some(allowed => path.startsWith(allowed));
}

/**
 * Rate-limited fetch with user agent
 */
async function rateLimitedFetch(url) {
  const now = Date.now();
  const timeSinceLastRequest = now - lastRequestTime;

  if (timeSinceLastRequest < RATE_LIMIT_MS) {
    await setTimeout(RATE_LIMIT_MS - timeSinceLastRequest);
  }

  lastRequestTime = Date.now();

  return fetch(url, {
    headers: {
      'User-Agent': 'Mozilla/5.0 (compatible; OmegaWatchBot/1.0; +http://45.61.58.125:7600)',
      'Accept': 'text/html,application/xhtml+xml',
      'Accept-Language': 'en-US,en;q=0.9'
    }
  });
}

/**
 * Get cached result or null
 */
function getCached(key) {
  const cached = cache.get(key);
  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
    return cached.data;
  }
  cache.delete(key);
  return null;
}

/**
 * Set cache
 */
function setCache(key, data) {
  cache.set(key, { data, timestamp: Date.now() });
}

/**
 * Extract listing data from HTML
 * Uses regex patterns to find structured data
 * @param {string} html - Page HTML content
 * @returns {array} Array of listing objects
 */
function extractListings(html) {
  const listings = [];

  // Look for JSON-LD structured data first (most reliable)
  const jsonLdMatch = html.match(/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g);
  if (jsonLdMatch) {
    for (const match of jsonLdMatch) {
      try {
        const jsonStr = match.replace(/<script[^>]*>/, '').replace(/<\/script>/, '');
        const data = JSON.parse(jsonStr);

        if (data['@type'] === 'Product' || data['@type'] === 'ItemList') {
          const items = data.itemListElement || [data];
          for (const item of items) {
            if (item.offers?.price) {
              listings.push({
                name: item.name || '',
                price: parseFloat(item.offers.price),
                currency: item.offers.priceCurrency || 'USD',
                url: item.url || '',
                condition: item.itemCondition?.replace('https://schema.org/', '') || 'unknown',
                seller: item.offers?.seller?.name || '',
                location: item.offers?.availableAtOrFrom?.address?.addressLocality || ''
              });
            }
          }
        }
      } catch (e) {
        // JSON parse failed, continue
      }
    }
  }

  // Fallback: Extract from HTML patterns
  if (listings.length === 0) {
    // Price patterns
    const pricePattern = /data-price="(\d+)"/g;
    const namePattern = /class="[^"]*article-title[^"]*"[^>]*>([^<]+)</g;

    let priceMatch;
    const prices = [];
    while ((priceMatch = pricePattern.exec(html)) !== null) {
      prices.push(parseFloat(priceMatch[1]));
    }

    let nameMatch;
    const names = [];
    while ((nameMatch = namePattern.exec(html)) !== null) {
      names.push(nameMatch[1].trim());
    }

    // Combine
    for (let i = 0; i < Math.min(prices.length, names.length); i++) {
      listings.push({
        name: names[i],
        price: prices[i],
        currency: 'USD',
        condition: 'unknown',
        source: 'html_extraction'
      });
    }
  }

  return listings;
}

/**
 * Scrape Omega listings from Chrono24
 * @param {string} reference - Watch reference number (optional)
 * @param {object} options - Scraping options
 * @returns {array} Array of listing objects
 */
export async function scrapeOmegaListings(reference = '', options = {}) {
  const { maxPages = 1, minPrice = 0, maxPrice = 100000 } = options;

  const path = reference
    ? `/omega/ref-${reference}.htm`
    : '/omega/index.htm';

  if (!isPathAllowed(path)) {
    console.warn(`Path not allowed by robots.txt: ${path}`);
    return [];
  }

  const cacheKey = `chrono24:${path}:${minPrice}:${maxPrice}`;
  const cached = getCached(cacheKey);
  if (cached) return cached;

  const allListings = [];

  for (let page = 1; page <= maxPages; page++) {
    try {
      const url = `${CHRONO24_BASE}${path}?priceFrom=${minPrice}&priceTo=${maxPrice}&pageSize=60&showpage=${page}`;

      console.log(`Scraping Chrono24 page ${page}: ${url}`);
      const response = await rateLimitedFetch(url);

      if (!response.ok) {
        console.warn(`Chrono24 returned ${response.status}`);
        break;
      }

      const html = await response.text();
      const listings = extractListings(html);

      if (listings.length === 0) {
        break; // No more results
      }

      allListings.push(...listings.map(listing => ({
        ...listing,
        reference: reference || 'unknown',
        source: 'chrono24',
        sourceUrl: url,
        scrapedAt: new Date().toISOString(),
        confidence: 4 // High confidence for direct scrape
      })));

    } catch (error) {
      console.error(`Chrono24 scrape error: ${error.message}`);
      break;
    }
  }

  setCache(cacheKey, allListings);
  return allListings;
}

/**
 * Get price statistics for a reference
 * @param {string} reference - Watch reference number
 * @returns {object} Price statistics
 */
export async function getPriceStats(reference) {
  const listings = await scrapeOmegaListings(reference, { maxPages: 1 });

  if (listings.length === 0) {
    return null;
  }

  const prices = listings.map(l => l.price).filter(p => p > 0).sort((a, b) => a - b);

  return {
    reference,
    count: prices.length,
    min: prices[0],
    max: prices[prices.length - 1],
    avg: Math.round(prices.reduce((a, b) => a + b, 0) / prices.length),
    median: prices[Math.floor(prices.length / 2)],
    lastUpdated: new Date().toISOString()
  };
}

/**
 * Search for watches by query
 * @param {string} query - Search query
 * @returns {array} Array of listing objects
 */
export async function searchWatches(query) {
  const path = `/search/index.htm`;
  const cacheKey = `chrono24:search:${query}`;
  const cached = getCached(cacheKey);
  if (cached) return cached;

  try {
    const url = `${CHRONO24_BASE}${path}?query=${encodeURIComponent(query)}&dosearch=true&searchexplain=1&pageSize=30`;

    const response = await rateLimitedFetch(url);
    if (!response.ok) {
      return [];
    }

    const html = await response.text();
    const listings = extractListings(html).map(listing => ({
      ...listing,
      query,
      source: 'chrono24_search',
      scrapedAt: new Date().toISOString()
    }));

    setCache(cacheKey, listings);
    return listings;

  } catch (error) {
    console.error(`Chrono24 search error: ${error.message}`);
    return [];
  }
}

/**
 * Clear expired cache entries
 */
export function clearExpiredCache() {
  const now = Date.now();
  for (const [key, value] of cache) {
    if (now - value.timestamp > CACHE_TTL) {
      cache.delete(key);
    }
  }
}

/**
 * Get scraper status
 */
export function getStatus() {
  return {
    cacheEntries: cache.size,
    lastRequest: new Date(lastRequestTime).toISOString(),
    rateLimitMs: RATE_LIMIT_MS,
    cacheTtlHours: CACHE_TTL / (60 * 60 * 1000)
  };
}

export default {
  scrapeOmegaListings,
  getPriceStats,
  searchWatches,
  clearExpiredCache,
  getStatus
};