`DW-Agents/dw-agents/amazon-realtime-scraper.ts`

279 lines
/**
 * Real-time Amazon Price Scraper
 * Works WITHOUT Product Advertising API
 * Scrapes actual Amazon product pages for real prices
 */

import fetch from 'node-fetch';
import * as cheerio from 'cheerio';

interface AmazonProduct {
  title: string;
  price: number;
  currency: string;
  url: string;
  asin?: string;
  rating?: number;
  reviewCount?: number;
  available: boolean;
}

interface SearchResult {
  query: string;
  products: AmazonProduct[];
  timestamp: Date;
}

/**
 * Search Amazon and scrape real product data
 */
export async function searchAmazonRealTime(query: string, maxResults: number = 10): Promise<SearchResult> {
  console.log(`🔍 Searching Amazon for: "${query}"`);

  const products: AmazonProduct[] = [];

  try {
    const searchQuery = encodeURIComponent(query);
    const searchUrl = `https://www.amazon.com/s?k=${searchQuery}`;

    const response = await fetch(searchUrl, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
        'Accept-Encoding': 'gzip, deflate, br',
        'DNT': '1',
        'Connection': 'keep-alive',
        'Upgrade-Insecure-Requests': '1'
      }
    });

    if (!response.ok) {
      console.error(`❌ Amazon returned status: ${response.status}`);
      return { query, products: [], timestamp: new Date() };
    }

    const html = await response.text();
    const $ = cheerio.load(html);

    // Find product cards
    $('div[data-component-type="s-search-result"]').each((idx, element) => {
      if (idx >= maxResults) return false;

      try {
        const $el = $(element);

        // Get ASIN
        const asin = $el.attr('data-asin') || '';

        // Get title
        const titleEl = $el.find('h2 a span, h2 span');
        const title = titleEl.first().text().trim();

        // Get price - try multiple selectors
        let price = 0;
        const priceWhole = $el.find('.a-price-whole').first().text().replace(/[^0-9]/g, '');
        const priceFraction = $el.find('.a-price-fraction').first().text().replace(/[^0-9]/g, '');

        if (priceWhole) {
          price = parseFloat(`${priceWhole}.${priceFraction || '00'}`);
        }

        // Alternative price selector
        if (!price) {
          const priceText = $el.find('.a-price .a-offscreen').first().text();
          const priceMatch = priceText.match(/[\d,]+\.?\d*/);
          if (priceMatch) {
            price = parseFloat(priceMatch[0].replace(/,/g, ''));
          }
        }

        // Get rating
        let rating = undefined;
        const ratingText = $el.find('.a-icon-star-small .a-icon-alt').text();
        const ratingMatch = ratingText.match(/(\d+\.?\d*)/);
        if (ratingMatch) {
          rating = parseFloat(ratingMatch[1]);
        }

        // Get review count
        let reviewCount = undefined;
        const reviewText = $el.find('span[aria-label]').filter((i, el) => {
          const label = $(el).attr('aria-label') || '';
          return label.includes('stars');
        }).parent().parent().find('span').last().text().trim();

        const reviewMatch = reviewText.match(/[\d,]+/);
        if (reviewMatch) {
          reviewCount = parseInt(reviewMatch[0].replace(/,/g, ''));
        }

        // Get product URL
        const productPath = $el.find('h2 a').attr('href') || '';
        const url = productPath ? `https://www.amazon.com${productPath}` : '';

        if (title && price > 0 && url) {
          products.push({
            title,
            price,
            currency: 'USD',
            url,
            asin,
            rating,
            reviewCount,
            available: true
          });

          console.log(`   ✅ ${title.substring(0, 50)}... - $${price.toFixed(2)}`);
        }

      } catch (err) {
        console.error('   ⚠️  Error parsing product:', err);
      }
    });

    console.log(`✅ Found ${products.length} products\n`);

  } catch (error: any) {
    console.error('❌ Error searching Amazon:', error.message);
  }

  return {
    query,
    products,
    timestamp: new Date()
  };
}

/**
 * Get product details by ASIN
 */
export async function getProductByAsin(asin: string): Promise<AmazonProduct | null> {
  console.log(`🔍 Fetching product: ${asin}`);

  try {
    const url = `https://www.amazon.com/dp/${asin}`;

    const response = await fetch(url, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5'
      }
    });

    if (!response.ok) {
      console.error(`❌ Amazon returned status: ${response.status}`);
      return null;
    }

    const html = await response.text();
    const $ = cheerio.load(html);

    // Get title
    const title = $('#productTitle').text().trim();

    // Get price
    let price = 0;
    const priceWhole = $('.a-price-whole').first().text().replace(/[^0-9]/g, '');
    const priceFraction = $('.a-price-fraction').first().text().replace(/[^0-9]/g, '');

    if (priceWhole) {
      price = parseFloat(`${priceWhole}.${priceFraction || '00'}`);
    }

    // Get rating
    let rating = undefined;
    const ratingText = $('.a-icon-star .a-icon-alt').first().text();
    const ratingMatch = ratingText.match(/(\d+\.?\d*)/);
    if (ratingMatch) {
      rating = parseFloat(ratingMatch[1]);
    }

    // Get review count
    let reviewCount = undefined;
    const reviewText = $('#acrCustomerReviewText').text();
    const reviewMatch = reviewText.match(/([\d,]+)/);
    if (reviewMatch) {
      reviewCount = parseInt(reviewMatch[0].replace(/,/g, ''));
    }

    if (title && price > 0) {
      console.log(`✅ ${title.substring(0, 50)}... - $${price.toFixed(2)}`);

      return {
        title,
        price,
        currency: 'USD',
        url,
        asin,
        rating,
        reviewCount,
        available: true
      };
    }

    console.log('⚠️  Could not extract product data');
    return null;

  } catch (error: any) {
    console.error('❌ Error fetching product:', error.message);
    return null;
  }
}

/**
 * Compare prices for office supplies
 */
export async function compareOfficeSupplyPrices(itemName: string): Promise<void> {
  console.log(`\n💰 Real-Time Price Comparison: ${itemName}\n`);
  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');

  const results = await searchAmazonRealTime(itemName, 5);

  if (results.products.length === 0) {
    console.log('⚠️  No products found');
    return;
  }

  // Sort by price (lowest first)
  results.products.sort((a, b) => a.price - b.price);

  console.log('📊 Live Amazon Prices:\n');

  results.products.forEach((product, idx) => {
    const badge = idx === 0 ? '🏆 BEST PRICE' : '';
    console.log(`${idx + 1}. ${product.title}`);
    console.log(`   💵 Price: $${product.price.toFixed(2)} ${badge}`);
    if (product.rating) {
      console.log(`   ⭐ Rating: ${product.rating}/5 (${product.reviewCount?.toLocaleString()} reviews)`);
    }
    console.log(`   🔗 URL: ${product.url}`);
    console.log('');
  });

  console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');
}

// Test function
if (require.main === module) {
  (async () => {
    console.log('🧪 Testing Real-Time Amazon Scraper\n');
    console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');
    console.log('✅ No API credentials needed!\n');
    console.log('✅ Scrapes live data from Amazon.com\n');
    console.log('━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n');

    // Test searches
    await compareOfficeSupplyPrices('copy paper');
    await new Promise(resolve => setTimeout(resolve, 2000));

    await compareOfficeSupplyPrices('black ink cartridge hp');
    await new Promise(resolve => setTimeout(resolve, 2000));

    await compareOfficeSupplyPrices('post-it notes');
  })();
}

export { AmazonProduct, SearchResult };