`crawler/hermes-specialist.js`

186 lines
const puppeteer = require('puppeteer');

class HermesSpecialistScraper {
  constructor() {
    this.baseUrl = 'https://auctions.yahoo.co.jp/search/search';

    // Specific Hermès bag models with Japanese names
    this.hermesModels = [
      'エルメス バーキン',        // Birkin
      'エルメス ケリー',          // Kelly
      'エルメス コンスタンス',    // Constance
      'エルメス ピコタン',        // Picotin
      'エルメス エヴリン',        // Evelyne
      'エルメス ガーデンパーティ', // Garden Party
      'エルメス ボリード',        // Bolide
      'エルメス リンディ',        // Lindy
      'エルメス ジプシエール',    // Jypsiere
      'エルメス エールバッグ',    // Herbag
      'エルメス トリム',          // Trim
      'エルメス フールトゥ',      // Fourre Tout
      'Hermes Birkin',
      'Hermes Kelly',
      'Hermes Constance'
    ];

    this.maxPages = 15; // More pages for high-value Hermès bags
  }

  async scrapeSearch(model, page = 1) {
    try {
      const browser = await puppeteer.launch({
        headless: 'new',
        args: ['--no-sandbox', '--disable-setuid-sandbox']
      });

      const browserPage = await browser.newPage();
      await browserPage.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

      const searchUrl = `${this.baseUrl}?p=${encodeURIComponent(model)}&b=${(page - 1) * 50 + 1}`;
      console.log(`Scraping Hermès specialist: ${searchUrl}`);

      await browserPage.goto(searchUrl, { waitUntil: 'networkidle2', timeout: 30000 });
      await browserPage.waitForSelector('.Product, .Product__item', { timeout: 10000 }).catch(() => {});

      const listings = await browserPage.evaluate(() => {
        const items = [];
        const productElements = document.querySelectorAll('.Product, .Product__item');

        productElements.forEach((item) => {
          try {
            const titleEl = item.querySelector('.Product__title, a');
            const title = titleEl?.textContent?.trim() || '';

            // Better price extraction - try multiple selectors
            let priceText = '';
            const priceSelectors = [
              '.Product__price',
              '.u-tax',
              '.Price',
              '.Product__priceValue',
              '[class*="price"]',
              '[class*="Price"]'
            ];

            for (const selector of priceSelectors) {
              const priceEl = item.querySelector(selector);
              if (priceEl && priceEl.textContent.match(/[\d,]+/)) {
                priceText = priceEl.textContent.trim();
                break;
              }
            }

            // Also try getting price from anywhere in the item
            if (!priceText) {
              const allText = item.textContent || '';
              const pricePattern = /(\d{1,3}(?:,\d{3})+)\s*円|¥\s*(\d{1,3}(?:,\d{3})+)/;
              const match = allText.match(pricePattern);
              if (match) {
                priceText = match[1] || match[2];
              }
            }

            const priceMatch = priceText.match(/[\d,]+/);
            const price = priceMatch ? parseInt(priceMatch[0].replace(/,/g, '')) : 0;

            const imgEl = item.querySelector('img');
            const image = imgEl?.src || '';

            const linkEl = item.querySelector('a');
            const url = linkEl?.href || '';

            const idMatch = url.match(/\/([a-z0-9]+)$/);
            const id = idMatch?.[1] || url;

            const timeEl = item.querySelector('.Product__time');
            const endTime = timeEl?.textContent?.trim() || '';

            // Try to extract size info from title
            const sizeMatch = title.match(/(\d+)\s*cm|サイズ\s*(\d+)/i);
            const size = sizeMatch ? (sizeMatch[1] || sizeMatch[2]) + 'cm' : '';

            // Try to extract color
            const colorMatch = title.match(/(ブラック|ホワイト|レッド|ブルー|グリーン|オレンジ|ピンク|ベージュ|グレー|Black|White|Red|Blue|Green|Orange|Pink|Beige|Grey|Etoupe|Gold|Rose)/i);
            const color = colorMatch ? colorMatch[1] : '';

            // Try to extract material
            const materialMatch = title.match(/(トゴ|エプソン|クレマンス|トリヨンクレマンス|スイフト|ヴォーエプソン|Togo|Epsom|Clemence|Swift|Box|Crocodile|Alligator|Lizard)/i);
            const material = materialMatch ? materialMatch[1] : '';

            // Detect condition
            let condition = 'Used';
            if (title.match(/未使用|新品|UNUSED|NEW/i)) {
              condition = 'New/Unused';
            } else if (title.match(/美品|極美品|EXCELLENT/i)) {
              condition = 'Excellent';
            } else if (title.match(/良品|GOOD/i)) {
              condition = 'Good';
            } else if (title.match(/難あり|ジャンク|JUNK|DAMAGED/i)) {
              condition = 'Fair/Damaged';
            }

            if (title && price > 0) {
              items.push({
                external_id: id,
                title,
                price_jpy: price,
                image_url: image,
                thumbnail_url: image,
                product_url: url,
                listing_type: 'auction',
                auction_end_date: endTime,
                condition,
                size,
                color,
                material
              });
            }
          } catch (e) {
            console.error('Error parsing Hermès item:', e);
          }
        });

        return items;
      });

      await browser.close();
      return listings;

    } catch (error) {
      console.error(`Error scraping Hermès specialist:`, error.message);
      return [];
    }
  }

  async scrapeAllModels() {
    const allListings = [];

    for (const model of this.hermesModels) {
      console.log(`\nScraping Hermès specialist for ${model}...`);

      for (let page = 1; page <= this.maxPages; page++) {
        const listings = await this.scrapeSearch(model, page);

        if (listings.length === 0) break;

        listings.forEach(listing => {
          listing.brand = 'Hermes';
          listing.source = 'yahoo_hermes_specialist';
          listing.model = model;
        });

        allListings.push(...listings);
        console.log(`Found ${listings.length} Hermès items on page ${page}`);

        // Longer delay for high-value scraping to avoid rate limits
        await new Promise(resolve => setTimeout(resolve, 4000));
      }
    }

    console.log(`\n✨ Hermès specialist crawl complete: ${allListings.length} total items`);
    return allListings;
  }
}

module.exports = HermesSpecialistScraper;