← back to Handbag Auth Nextjs

test_scraper.js

167 lines

const puppeteer = require('puppeteer');

async function testSite(siteName, url, waitSelectors) {
  console.log(`\n=== Testing ${siteName} ===`);
  console.log(`URL: ${url}`);
  
  const browser = await puppeteer.launch({
    headless: true,
    args: ['--no-sandbox', '--disable-setuid-sandbox']
  });
  
  try {
    const page = await browser.newPage();
    await page.setViewport({ width: 1920, height: 1080 });
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
    
    await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 });
    
    // Scroll to trigger lazy loading
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight / 2);
    });
    await new Promise(r => setTimeout(r, 3000));
    
    // Try multiple selector patterns
    let foundSelector = null;
    for (const selector of waitSelectors) {
      try {
        await page.waitForSelector(selector, { timeout: 3000 });
        foundSelector = selector;
        console.log(`✓ Found elements with selector: ${selector}`);
        break;
      } catch (e) {
        // Try next selector
      }
    }
    
    if (!foundSelector) {
      console.log('✗ No product containers found with any selector');
      // Dump page structure
      const structure = await page.evaluate(() => {
        const body = document.body;
        const allElements = body.querySelectorAll('*');
        const classNames = new Set();
        const dataAttrs = new Set();
        
        allElements.forEach(el => {
          if (el.className && typeof el.className === 'string') {
            el.className.split(' ').forEach(c => {
              if (c.includes('product') || c.includes('item') || c.includes('card')) {
                classNames.add(c);
              }
            });
          }
          Array.from(el.attributes || []).forEach(attr => {
            if (attr.name.startsWith('data-')) {
              dataAttrs.add(attr.name);
            }
          });
        });
        
        return {
          uniqueProductClasses: Array.from(classNames).slice(0, 20),
          uniqueDataAttrs: Array.from(dataAttrs).slice(0, 20)
        };
      });
      console.log('Page structure:', JSON.stringify(structure, null, 2));
    } else {
      // Analyze the found elements
      const analysis = await page.evaluate((selector) => {
        const containers = document.querySelectorAll(selector);
        if (containers.length === 0) return { error: 'No containers' };
        
        const first = containers[0];
        
        // Look for title
        const titleSelectors = ['h2', 'h3', 'a[href*="product"]', '[class*="title"]', '[class*="name"]'];
        let titleEl = null;
        let titleSelector = null;
        for (const ts of titleSelectors) {
          titleEl = first.querySelector(ts);
          if (titleEl && titleEl.textContent.trim()) {
            titleSelector = ts;
            break;
          }
        }
        
        // Look for price
        const priceSelectors = ['[class*="price"]', '[data-testid*="price"]', 'span', 'p'];
        let priceEl = null;
        let priceSelector = null;
        for (const ps of priceSelectors) {
          const candidates = first.querySelectorAll(ps);
          for (const candidate of candidates) {
            if (candidate.textContent.includes('$')) {
              priceEl = candidate;
              priceSelector = ps;
              break;
            }
          }
          if (priceEl) break;
        }
        
        // Look for image
        const img = first.querySelector('img');
        
        // Look for link
        const link = first.querySelector('a[href*="product"], a[href*="item"]') || first.querySelector('a');
        
        return {
          containerCount: containers.length,
          containerHTML: first.outerHTML.substring(0, 1500),
          title: {
            selector: titleSelector,
            text: titleEl?.textContent.trim().substring(0, 100)
          },
          price: {
            selector: priceSelector,
            text: priceEl?.textContent.trim()
          },
          image: {
            found: !!img,
            src: img?.src?.substring(0, 100)
          },
          link: {
            found: !!link,
            href: link?.href?.substring(0, 100)
          }
        };
      }, foundSelector);
      
      console.log('Analysis:', JSON.stringify(analysis, null, 2));
    }
    
  } catch (error) {
    console.error(`Error: ${error.message}`);
  } finally {
    await browser.close();
  }
}

(async () => {
  await testSite('Fashionphile', 'https://www.fashionphile.com/shop/bags?designers=HERMÈS', [
    '[data-testid="product-card"]',
    '.product-tile',
    '.product-card',
    '[class*="ProductCard"]',
    'article',
    'li[class*="product"]'
  ]);
  
  await testSite('Rebag', 'https://www.rebag.com/shop?brand=HERMÈS', [
    '[data-testid="product-card"]',
    '[data-testid="product"]',
    '.product-item',
    'article',
    '[class*="ProductCard"]'
  ]);
  
  await testSite('TheRealReal', 'https://www.therealreal.com/designers/hermès/handbags', [
    '[data-testid="product-card"]',
    '.product-tile',
    'article[class*="product"]',
    '[class*="ProductCard"]'
  ]);
})();