← back to Handbag Authentication

crawler/goodwill-scraper.js

154 lines

const puppeteer = require('puppeteer');

/**
 * Goodwill.com Scraper
 * ShopGoodwill.com - online auction site for Goodwill donations
 * Often has designer bags donated unknowingly
 */
class GoodwillScraper {
  constructor() {
    this.baseUrl = 'https://www.shopgoodwill.com';
  }

  async scrapeSearch(brand, page = 1) {
    try {
      const browser = await puppeteer.launch({
        headless: 'new',
        args: ['--no-sandbox', '--disable-setuid-sandbox']
      });

      const browserPage = await browser.newPage();
      await browserPage.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');

      const searchUrl = `${this.baseUrl}/listings?q=${encodeURIComponent(brand + ' handbag')}&page=${page}&st=categorysearch&sg=&c=&s=&lp=0&hp=9999&sbn=false&spo=false&snpo=false&socs=false&sd=false&sca=false&caed=12/9/2024&cadb=7&scs=false&sis=false&col=1&p=1&ps=40&desc=false&ss=0&UseBuyerPrefs=true`;
      console.log(`Scraping Goodwill: ${searchUrl}`);

      await browserPage.goto(searchUrl, { waitUntil: 'networkidle2', timeout: 30000 });
      await browserPage.waitForSelector('.product-card, .item, [class*="product"]', { timeout: 10000 }).catch(() => {});

      const listings = await browserPage.evaluate(() => {
        const items = [];
        const itemElements = document.querySelectorAll('.product-card, .item, [class*="product-item"]');

        itemElements.forEach((item) => {
          try {
            const titleEl = item.querySelector('.product-title, .title, h3, .item-title');
            const title = titleEl?.textContent?.trim() || '';

            // Price extraction - Goodwill shows current bid
            let price = 0;
            const priceSelectors = [
              '.current-price',
              '.price',
              '[class*="bid"]',
              '[class*="price"]'
            ];

            for (const selector of priceSelectors) {
              const priceEl = item.querySelector(selector);
              if (priceEl) {
                const priceText = priceEl.textContent.trim();
                const priceMatch = priceText.match(/\$?([\d,]+\.?\d*)/);
                if (priceMatch) {
                  price = parseFloat(priceMatch[1].replace(/,/g, ''));
                  break;
                }
              }
            }

            const imgEl = item.querySelector('img');
            const image = imgEl?.src || imgEl?.getAttribute('data-src') || '';

            const linkEl = item.querySelector('a');
            let url = linkEl?.href || '';
            if (url && !url.startsWith('http')) {
              url = 'https://www.shopgoodwill.com' + url;
            }

            const idMatch = url.match(/itemid=(\d+)/);
            const id = idMatch?.[1] || url;

            const timeEl = item.querySelector('.time-left, .ends-at, [class*="time"]');
            const endTime = timeEl?.textContent?.trim() || '';

            // Condition from Goodwill
            const conditionEl = item.querySelector('.condition, [class*="condition"]');
            const condition = conditionEl?.textContent?.trim() || 'Used';

            // Only include if it mentions designer brands
            const isDesignerBag = title.match(/Louis Vuitton|Gucci|Chanel|Hermes|Prada|Dior|Fendi|Balenciaga|Celine|Bottega|Coach|Kate Spade|Michael Kors|Designer|Luxury|Authentic/i);

            if (title && price > 0 && isDesignerBag) {
              items.push({
                external_id: `goodwill_${id}`,
                title,
                price_jpy: Math.round(price * 150), // Convert USD to JPY estimate
                price_usd: price,
                image_url: image,
                thumbnail_url: image,
                product_url: url,
                listing_type: 'auction',
                auction_end_date: endTime,
                condition
              });
            }
          } catch (e) {
            console.error('Error parsing Goodwill item:', e);
          }
        });

        return items;
      });

      await browser.close();
      return listings;

    } catch (error) {
      console.error(`Error scraping Goodwill page ${page}:`, error.message);
      return [];
    }
  }

  async scrapeAllBrands() {
    const allListings = [];
    const brands = [
      'Louis Vuitton',
      'Gucci',
      'Chanel',
      'Hermes',
      'Prada',
      'Dior',
      'Fendi',
      'Coach',
      'Kate Spade',
      'Michael Kors'
    ];

    for (const brand of brands) {
      console.log(`\nScraping Goodwill for ${brand}...`);

      for (let page = 1; page <= 5; page++) {
        const listings = await this.scrapeSearch(brand, page);

        if (listings.length === 0) break;

        listings.forEach(listing => {
          listing.source = 'goodwill';
          // Try to extract brand from title
          const brandMatch = listing.title.match(/Louis Vuitton|Gucci|Chanel|Hermes|Prada|Dior|Fendi|Coach|Kate Spade|Michael Kors/i);
          listing.brand = brandMatch ? brandMatch[0] : brand;
        });

        allListings.push(...listings);
        console.log(`Found ${listings.length} items on page ${page}`);

        await new Promise(resolve => setTimeout(resolve, 3000));
      }
    }

    return allListings;
  }
}

module.exports = GoodwillScraper;