`crawler/ebay-japan.js`

145 lines
const puppeteer = require('puppeteer');

class EbayJapanScraper {
  constructor() {
    this.baseUrl = 'https://www.ebay.com/sch/i.html';
    this.brands = ['Gucci', 'Louis Vuitton', 'Chanel', 'Hermes', 'Prada', 'Dior', 'Fendi', 'Balenciaga', 'Celine', 'Bottega Veneta'];
    this.maxPages = 5;
  }

  async scrapeSearch(brand, page = 1) {
    // Search for items from Japan that ship to US
    const searchUrl = `${this.baseUrl}?_nkw=${encodeURIComponent(brand + ' bag')}&_sacat=0&LH_ItemCondition=3000|4000&_sop=10&_pgn=${page}&LH_PrefLoc=2`;

    try {
      const browser = await puppeteer.launch({
        headless: 'new',
        args: ['--no-sandbox', '--disable-setuid-sandbox']
      });

      const browserPage = await browser.newPage();
      await browserPage.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

      console.log(`Scraping eBay Japan: ${searchUrl}`);
      await browserPage.goto(searchUrl, { waitUntil: 'networkidle2', timeout: 30000 });

      // Wait for search results
      await browserPage.waitForSelector('.s-item, .srp-results li', { timeout: 10000 }).catch(() => {});

      const listings = await browserPage.evaluate(() => {
        const items = [];
        const productElements = document.querySelectorAll('.s-item');

        productElements.forEach((item) => {
          try {
            // Skip sponsored items
            if (item.classList.contains('s-item--ad')) return;

            // Extract image
            const imgEl = item.querySelector('img');
            const image = imgEl?.src || '';

            // Extract title
            const titleEl = item.querySelector('.s-item__title');
            const title = titleEl?.textContent?.trim() || '';

            // Skip "Shop on eBay" banner
            if (title.includes('Shop on eBay')) return;

            // Extract price
            const priceEl = item.querySelector('.s-item__price');
            const priceText = priceEl?.textContent?.trim() || '';

            // Parse USD price
            const priceMatch = priceText.match(/\$?([\d,]+\.?\d*)/);
            const priceUSD = priceMatch ? parseFloat(priceMatch[1].replace(/,/g, '')) : 0;

            // Estimate JPY price (USD * 150)
            const priceJPY = Math.round(priceUSD * 150);

            // Extract URL
            const linkEl = item.querySelector('.s-item__link');
            const url = linkEl?.href || '';

            // Extract ID from URL
            const idMatch = url.match(/\/itm\/(\d+)/);
            const id = idMatch?.[1] || url;

            // Extract condition
            const conditionEl = item.querySelector('.SECONDARY_INFO');
            const condition = conditionEl?.textContent?.trim() || 'Pre-Owned';

            // Extract shipping info to verify it's from Japan
            const shippingEl = item.querySelector('.s-item__shipping, .s-item__logisticsCost');
            const shippingText = shippingEl?.textContent?.trim() || '';
            const fromJapan = shippingText.toLowerCase().includes('japan') ||
                            shippingText.includes('+$') ||
                            item.textContent.includes('Japan');

            // Extract listing type (auction or buy it now)
            const listingType = item.querySelector('.s-item__purchase-options-with-icon')?.textContent?.includes('auction') ? 'auction' : 'buy';

            if (title && priceUSD > 0 && fromJapan) {
              items.push({
                external_id: id,
                title,
                price_jpy: priceJPY,
                price_usd: priceUSD,
                image_url: image,
                thumbnail_url: image,
                product_url: url,
                condition,
                listing_type: listingType
              });
            }
          } catch (e) {
            console.error('Error parsing eBay item:', e);
          }
        });

        return items;
      });

      await browser.close();
      return listings;

    } catch (error) {
      console.error(`Error scraping eBay Japan page ${page}:`, error.message);
      return [];
    }
  }

  async scrapeAllBrands() {
    const allListings = [];

    for (const brand of this.brands) {
      console.log(`\nScraping eBay Japan for ${brand}...`);

      for (let page = 1; page <= this.maxPages; page++) {
        const listings = await this.scrapeSearch(brand, page);

        if (listings.length === 0) {
          console.log(`No more results for ${brand} at page ${page}`);
          break;
        }

        // Add brand and source info
        listings.forEach(listing => {
          listing.brand = brand;
          listing.source = 'ebay_japan';
        });

        allListings.push(...listings);
        console.log(`Found ${listings.length} items on page ${page}`);

        // Rate limiting
        await new Promise(resolve => setTimeout(resolve, 3000 + Math.random() * 2000));
      }
    }

    return allListings;
  }
}

module.exports = EbayJapanScraper;