`crawler/mercari.js`

105 lines
const puppeteer = require('puppeteer');

class MercariJPScraper {
  constructor() {
    this.baseUrl = 'https://jp.mercari.com/search';
    this.brands = ['グッチ バッグ', 'ルイヴィトン バッグ', 'シャネル バッグ', 'エルメス バッグ'];
  }

  async scrapeSearch(brand, page = 1) {
    try {
      const browser = await puppeteer.launch({
        headless: 'new',
        args: ['--no-sandbox', '--disable-setuid-sandbox']
      });

      const browserPage = await browser.newPage();
      await browserPage.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

      const searchUrl = `${this.baseUrl}?keyword=${encodeURIComponent(brand)}&page=${page}`;
      console.log(`Scraping Mercari JP: ${searchUrl}`);

      await browserPage.goto(searchUrl, { waitUntil: 'networkidle2', timeout: 30000 });
      await browserPage.waitForSelector('[data-testid="item-cell"]', { timeout: 10000 }).catch(() => {});

      const listings = await browserPage.evaluate(() => {
        const items = [];
        const productElements = document.querySelectorAll('[data-testid="item-cell"]');

        productElements.forEach((item) => {
          try {
            const titleEl = item.querySelector('span');
            const title = titleEl?.textContent?.trim() || '';

            const priceEl = item.querySelector('[data-testid="price"]');
            const priceText = priceEl?.textContent?.trim() || '';
            const priceMatch = priceText.match(/[\d,]+/);
            const price = priceMatch ? parseInt(priceMatch[0].replace(/,/g, '')) : 0;

            const imgEl = item.querySelector('img');
            const image = imgEl?.src || '';

            const linkEl = item.querySelector('a');
            const url = linkEl?.href || '';

            const idMatch = url.match(/\/items\/([a-zA-Z0-9]+)/);
            const id = idMatch?.[1] || url;

            if (title && price > 0) {
              items.push({
                external_id: id,
                title,
                price_jpy: price,
                image_url: image,
                thumbnail_url: image,
                product_url: url,
                listing_type: 'buy',
                condition: 'Used'
              });
            }
          } catch (e) {
            console.error('Error parsing Mercari item:', e);
          }
        });

        return items;
      });

      await browser.close();
      return listings;

    } catch (error) {
      console.error(`Error scraping Mercari JP:`, error.message);
      return [];
    }
  }

  async scrapeAllBrands() {
    const allListings = [];

    for (const brand of this.brands) {
      console.log(`\nScraping Mercari JP for ${brand}...`);

      for (let page = 1; page <= 5; page++) {
        const listings = await this.scrapeSearch(brand, page);

        if (listings.length === 0) break;

        listings.forEach(listing => {
          listing.brand = brand;
          listing.source = 'mercari_jp';
        });

        allListings.push(...listings);
        console.log(`Found ${listings.length} items on page ${page}`);

        await new Promise(resolve => setTimeout(resolve, 2500));
      }
    }

    return allListings;
  }
}

module.exports = MercariJPScraper;