`services/wayback-scraper.js`

311 lines
/**
 * Wayback Machine Price Scraper Service
 * Fetches historical watch prices from archived dealer pages
 */

import { setTimeout } from 'timers/promises';

const WAYBACK_CDX_API = 'https://web.archive.org/cdx/search/cdx';
const WAYBACK_WEB_API = 'https://web.archive.org/web';

// Rate limiting: 1 request per second
const RATE_LIMIT_MS = 1000;
let lastRequestTime = 0;

// In-memory cache
const cache = new Map();
const CACHE_TTL = 60 * 60 * 1000; // 1 hour

/**
 * Rate-limited fetch
 */
async function rateLimitedFetch(url, options = {}) {
  const now = Date.now();
  const timeSinceLastRequest = now - lastRequestTime;

  if (timeSinceLastRequest < RATE_LIMIT_MS) {
    await setTimeout(RATE_LIMIT_MS - timeSinceLastRequest);
  }

  lastRequestTime = Date.now();
  return fetch(url, options);
}

/**
 * Get cached result or null
 */
function getCached(key) {
  const cached = cache.get(key);
  if (cached && Date.now() - cached.timestamp < CACHE_TTL) {
    return cached.data;
  }
  cache.delete(key);
  return null;
}

/**
 * Set cache
 */
function setCache(key, data) {
  cache.set(key, { data, timestamp: Date.now() });
}

/**
 * Query Wayback Machine CDX API for snapshots of a URL
 * @param {string} url - The URL to search for
 * @param {object} options - Query options
 * @returns {array} Array of snapshot records
 */
export async function getSnapshots(url, options = {}) {
  const {
    from = '20100101',
    to = new Date().toISOString().slice(0, 10).replace(/-/g, ''),
    limit = 50,
    filter = 'statuscode:200'
  } = options;

  const cacheKey = `snapshots:${url}:${from}:${to}`;
  const cached = getCached(cacheKey);
  if (cached) return cached;

  const params = new URLSearchParams({
    url,
    output: 'json',
    from,
    to,
    limit,
    filter,
    fl: 'timestamp,original,statuscode,digest'
  });

  try {
    const response = await rateLimitedFetch(`${WAYBACK_CDX_API}?${params}`);

    if (!response.ok) {
      throw new Error(`CDX API error: ${response.status}`);
    }

    const data = await response.json();

    // First row is headers
    const headers = data[0];
    const records = data.slice(1).map(row => {
      const record = {};
      headers.forEach((header, i) => {
        record[header] = row[i];
      });
      return record;
    });

    setCache(cacheKey, records);
    return records;

  } catch (error) {
    console.error('Wayback CDX error:', error.message);
    return [];
  }
}

/**
 * Fetch archived page content
 * @param {string} url - Original URL
 * @param {string} timestamp - Wayback timestamp (YYYYMMDDhhmmss)
 * @returns {string} Page HTML content
 */
export async function fetchArchivedPage(url, timestamp) {
  const cacheKey = `page:${url}:${timestamp}`;
  const cached = getCached(cacheKey);
  if (cached) return cached;

  const waybackUrl = `${WAYBACK_WEB_API}/${timestamp}/${url}`;

  try {
    const response = await rateLimitedFetch(waybackUrl);

    if (!response.ok) {
      throw new Error(`Wayback fetch error: ${response.status}`);
    }

    const html = await response.text();
    setCache(cacheKey, html);
    return html;

  } catch (error) {
    console.error('Wayback fetch error:', error.message);
    return null;
  }
}

/**
 * Extract price from HTML content
 * Supports common dealer page formats
 * @param {string} html - Page HTML
 * @param {object} patterns - Custom extraction patterns
 * @returns {number|null} Extracted price or null
 */
export function extractPrice(html, patterns = {}) {
  if (!html) return null;

  // Common price patterns
  const pricePatterns = [
    // USD formats
    /\$\s*([\d,]+(?:\.\d{2})?)/g,
    // "Price: $X,XXX"
    /price[:\s]*\$\s*([\d,]+(?:\.\d{2})?)/gi,
    // "USD X,XXX"
    /USD\s*([\d,]+(?:\.\d{2})?)/gi,
    // Schema.org price
    /"price":\s*"?([\d,]+(?:\.\d{2})?)"?/g,
    // data-price attribute
    /data-price="([\d,]+(?:\.\d{2})?)"/gi,
    // Custom patterns
    ...(patterns.custom || [])
  ];

  const prices = [];

  for (const pattern of pricePatterns) {
    const matches = html.matchAll(pattern);
    for (const match of matches) {
      const price = parseFloat(match[1].replace(/,/g, ''));
      if (price > 0 && price < 10000000) { // Sanity check
        prices.push(price);
      }
    }
  }

  if (prices.length === 0) return null;

  // Return median price if multiple found
  prices.sort((a, b) => a - b);
  return prices[Math.floor(prices.length / 2)];
}

/**
 * Scrape historical prices for a watch from Wayback Machine
 * @param {string} watchRef - Watch reference number
 * @param {array} dealerUrls - Array of dealer URLs to search
 * @returns {array} Array of { date, price, source, confidence }
 */
export async function scrapeHistoricalPrices(watchRef, dealerUrls = []) {
  const results = [];

  // Default dealer URLs if none provided
  const urls = dealerUrls.length > 0 ? dealerUrls : [
    `https://www.chrono24.com/omega/ref-${watchRef}.htm`,
    `https://www.bobswatches.com/omega/${watchRef}`,
    `https://www.omegawatches.com/${watchRef}`
  ];

  for (const url of urls) {
    try {
      // Get snapshots
      const snapshots = await getSnapshots(url, { limit: 20 });

      for (const snapshot of snapshots) {
        // Fetch archived page
        const html = await fetchArchivedPage(url, snapshot.timestamp);
        if (!html) continue;

        // Extract price
        const price = extractPrice(html);
        if (!price) continue;

        // Parse date from timestamp
        const timestamp = snapshot.timestamp;
        const date = new Date(
          timestamp.slice(0, 4),
          parseInt(timestamp.slice(4, 6)) - 1,
          timestamp.slice(6, 8)
        );

        results.push({
          date: date.toISOString().slice(0, 10),
          year: date.getFullYear(),
          price,
          source: 'wayback',
          sourceUrl: url,
          confidence: 3, // Medium confidence for scraped data
          timestamp: snapshot.timestamp
        });
      }

    } catch (error) {
      console.error(`Error scraping ${url}:`, error.message);
    }
  }

  // Deduplicate by date (keep highest confidence)
  const byDate = new Map();
  for (const result of results) {
    const existing = byDate.get(result.date);
    if (!existing || result.confidence > existing.confidence) {
      byDate.set(result.date, result);
    }
  }

  return Array.from(byDate.values()).sort((a, b) =>
    new Date(a.date) - new Date(b.date)
  );
}

/**
 * Get price trends from historical data
 * @param {array} priceData - Array of price records
 * @returns {object} Trend analysis
 */
export function analyzePriceTrends(priceData) {
  if (!priceData || priceData.length < 2) {
    return { trend: 'insufficient_data', change: 0 };
  }

  const sorted = [...priceData].sort((a, b) =>
    new Date(a.date) - new Date(b.date)
  );

  const oldest = sorted[0];
  const newest = sorted[sorted.length - 1];

  const priceChange = newest.price - oldest.price;
  const percentChange = (priceChange / oldest.price) * 100;

  const years = (new Date(newest.date) - new Date(oldest.date)) /
    (365.25 * 24 * 60 * 60 * 1000);

  const annualizedReturn = years > 0 ?
    (Math.pow(newest.price / oldest.price, 1 / years) - 1) * 100 : 0;

  return {
    trend: priceChange > 0 ? 'up' : priceChange < 0 ? 'down' : 'stable',
    oldestPrice: oldest.price,
    oldestDate: oldest.date,
    newestPrice: newest.price,
    newestDate: newest.date,
    absoluteChange: priceChange,
    percentChange: percentChange.toFixed(2),
    annualizedReturn: annualizedReturn.toFixed(2),
    dataPoints: priceData.length
  };
}

/**
 * Clear expired cache entries
 */
export function clearExpiredCache() {
  const now = Date.now();
  for (const [key, value] of cache) {
    if (now - value.timestamp > CACHE_TTL) {
      cache.delete(key);
    }
  }
}

export default {
  getSnapshots,
  fetchArchivedPage,
  extractPrice,
  scrapeHistoricalPrices,
  analyzePriceTrends,
  clearExpiredCache
};