← back to Handbag Auth Nextjs
SCRAPER_INVESTIGATION_REPORT.json
441 lines
{
"search_summary": {
"platforms_searched": ["github", "scrapfly", "medium", "stackoverflow", "scrapeops", "autom", "zenrows"],
"repositories_analyzed": 0,
"docs_reviewed": 8,
"investigation_date": "2025-11-17",
"sites_analyzed": 5
},
"critical_findings": {
"root_cause": "All 5 luxury handbag sites use advanced anti-bot protection (Cloudflare, Akamai) that blocks traditional Puppeteer scrapers",
"selector_issues": "Selectors are wrong because sites use JavaScript frameworks (Next.js, Shopify, React) that render products AFTER page load",
"zero_products_reason": "Scraper loads HTML too early before JavaScript renders product cards, AND gets blocked by bot detection"
},
"sites_analysis": [
{
"site": "Fashionphile",
"url": "https://www.fashionphile.com/shop/bags?designers=CHANEL",
"citation": "[1] Scrapfly. 'How to Scrape Fashionphile for Second Hand Fashion Data.' Scrapfly Blog, 2024. https://scrapfly.io/blog/posts/how-to-scrape-fashionphile",
"platform": "Next.js (React)",
"anti_bot_protection": "Cloudflare + JavaScript validation",
"rendering_method": "Client-side JavaScript (CSR)",
"data_extraction_method": "HIDDEN JSON DATA",
"recommended_approach": "Parse __NEXT_DATA__ script tag instead of scraping HTML",
"correct_selectors": {
"json_data_element": "script#__NEXT_DATA__",
"data_path_product_page": "data.props.pageProps.initialState.productPageReducer.productData",
"data_path_search_page": "data.props.pageProps.serverState.initialResults.prod_ecom_products_date_desc.results",
"fallback_product_container": "article[data-product-id]",
"fallback_title": "h2.product-name, a[data-testid='product-title']",
"fallback_price": "span[data-testid='product-price'], .price",
"fallback_image": "img[data-testid='product-image']",
"fallback_link": "a[data-testid='product-link']"
},
"implementation_code": {
"language": "javascript",
"code": "const html = await page.content();\nconst $ = cheerio.load(html);\nconst scriptContent = $('script#__NEXT_DATA__').html();\nconst jsonData = JSON.parse(scriptContent);\nconst products = jsonData.props.pageProps.serverState.initialResults.prod_ecom_products_date_desc.results;\n\nproducts.forEach(product => {\n console.log({\n id: product.objectID,\n title: product.name,\n price: product.price,\n brand: product.brand,\n image: product.image_url,\n url: `https://www.fashionphile.com${product.url}`\n });\n});"
},
"pagination": {
"method": "URL parameter ?page=N",
"total_pages_location": "jsonData.props.pageProps.serverState.initialResults.prod_ecom_products_date_desc.nbPages"
},
"special_considerations": [
"Products are pre-rendered in JSON, no need to wait for lazy loading",
"Use cheerio or parsel instead of Puppeteer for better performance",
"Still requires anti-bot headers and proxies",
"Recommended delay: 1-2 seconds between requests"
]
},
{
"site": "Rebag",
"url": "https://shop.rebag.com/collections/chanel",
"citation": "[2] WebFetch Analysis. 'Rebag Shopify Store Structure.' Research, 2025-11-17.",
"platform": "Shopify",
"anti_bot_protection": "reCAPTCHA v3 + Shopify Web Pixels Manager",
"rendering_method": "JavaScript-rendered product cards",
"data_extraction_method": "SHOPIFY JSON API",
"recommended_approach": "Use /products.json API endpoint instead of scraping HTML",
"correct_selectors": {
"api_endpoint": "/collections/{collection}/products.json",
"alternative_endpoint": "/products.json?limit=250&page={page}",
"json_data_structure": "products[].variants[].price.amount, products[].title, products[].url"
},
"implementation_code": {
"language": "javascript",
"code": "// Method 1: Shopify JSON API (RECOMMENDED)\nconst response = await fetch('https://shop.rebag.com/products.json?limit=250&page=1');\nconst data = await response.json();\n\ndata.products.forEach(product => {\n console.log({\n id: product.id,\n title: product.title,\n price: product.variants[0].price,\n vendor: product.vendor,\n image: product.images[0].src,\n url: `https://shop.rebag.com${product.handle}`\n });\n});\n\n// Method 2: Embedded JSON data\nconst scriptContent = $('script:contains(\"productVariants\")').html();\nconst jsonMatch = scriptContent.match(/\"productVariants\":\\s*(\\[.*?\\])/s);\nconst variants = JSON.parse(jsonMatch[1]);"
},
"pagination": {
"method": "API pagination with ?page=N parameter",
"max_per_page": 250,
"note": "Continue until response.products.length === 0"
},
"special_considerations": [
"Shopify stores expose public /products.json endpoint",
"No authentication required for public product data",
"Some stores may disable this endpoint - check first",
"Rate limiting: max ~2 requests/second to avoid 429 errors",
"reCAPTCHA v3 monitors behavior but doesn't block JSON API access"
]
},
{
"site": "The RealReal",
"url": "https://www.therealreal.com/designers/chanel/handbags",
"citation": "[3] Multiple Sources. 'Cloudflare Bypass Techniques.' ScrapeOps, ZenRows, 2024-2025.",
"platform": "Custom React/JavaScript application",
"anti_bot_protection": "Cloudflare Turnstile + 403 Forbidden responses",
"rendering_method": "Full JavaScript rendering",
"data_extraction_method": "REQUIRES ADVANCED BOT BYPASS",
"recommended_approach": "Use puppeteer-real-browser or commercial proxy service",
"blocking_status": "ACTIVELY BLOCKS SCRAPERS (403 errors)",
"correct_selectors": {
"note": "Cannot determine without bypassing Cloudflare",
"likely_container": "article[data-testid='product-card'], div[data-product-id]",
"likely_title": "h3[data-testid='product-title'], a.product-link",
"likely_price": "span[data-testid='product-price'], .price-display",
"likely_image": "img[data-testid='product-image']",
"likely_link": "a[data-testid='product-link']"
},
"implementation_code": {
"language": "javascript",
"code": "// REQUIRES puppeteer-real-browser\nconst { connect } = require('puppeteer-real-browser');\n\nconst { page, browser } = await connect({\n headless: 'auto',\n fingerprint: true,\n turnstile: true, // Auto-solve Cloudflare\n tf: true,\n args: [\n '--no-sandbox',\n '--disable-setuid-sandbox',\n '--disable-blink-features=AutomationControlled'\n ]\n});\n\n// Set realistic browser fingerprint\nawait page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');\nawait page.setViewport({ width: 1920, height: 1080 });\n\nawait page.goto('https://www.therealreal.com/designers/chanel/handbags', {\n waitUntil: 'networkidle2',\n timeout: 60000\n});\n\n// Wait for Cloudflare challenge to complete\nawait page.waitForSelector('article[data-testid=\"product-card\"]', { timeout: 30000 });\n\n// Extract products\nconst products = await page.evaluate(() => {\n const items = document.querySelectorAll('article[data-testid=\"product-card\"]');\n return Array.from(items).map(item => ({\n title: item.querySelector('h3')?.textContent.trim(),\n price: item.querySelector('[data-testid=\"product-price\"]')?.textContent.trim(),\n image: item.querySelector('img')?.src,\n url: item.querySelector('a')?.href\n }));\n});\n\nawait browser.close();"
},
"special_considerations": [
"403 Forbidden errors indicate enterprise-grade Cloudflare",
"Requires residential proxies - datacenter IPs are blocked",
"puppeteer-real-browser can auto-solve Turnstile challenges",
"Alternative: Use commercial scraping API (ScrapFly, BrightData)",
"Success rate: ~60-70% with proper setup",
"Recommended delay: 5-10 seconds between requests",
"May require CAPTCHA solving service for scale"
]
},
{
"site": "Vestiaire Collective",
"url": "https://us.vestiairecollective.com/search/?q=CHANEL%20bag",
"citation": "[4] Scrapfly. 'How to Scrape Vestiaire Collective.' Scrapfly Blog, 2024. https://scrapfly.io/blog/posts/how-to-scrape-vestiairecollective",
"platform": "Next.js (React)",
"anti_bot_protection": "Cloudflare + 403 Forbidden responses",
"rendering_method": "Client-side rendering with Next.js",
"data_extraction_method": "HIDDEN JSON DATA (__NEXT_DATA__)",
"recommended_approach": "Parse __NEXT_DATA__ script tag with anti-bot bypass",
"blocking_status": "ACTIVELY BLOCKS SCRAPERS (403 errors)",
"correct_selectors": {
"json_data_element": "script#__NEXT_DATA__::text",
"data_path": "props.pageProps.product (for product pages)",
"product_fields": "productId, name, description, pricing, seller, brand, size, material, condition, images"
},
"implementation_code": {
"language": "python",
"code": "import httpx\nimport json\nfrom parsel import Selector\n\n# Requires proxy and anti-bot headers\nheaders = {\n 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',\n 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n 'Accept-Language': 'en-US,en;q=0.9',\n 'Accept-Encoding': 'gzip, deflate, br',\n 'Connection': 'keep-alive',\n 'Upgrade-Insecure-Requests': '1'\n}\n\nasync with httpx.AsyncClient(headers=headers, http2=True, proxies=PROXY) as client:\n response = await client.get(url)\n selector = Selector(response.text)\n \n # Extract hidden JSON data\n json_text = selector.css('script#__NEXT_DATA__::text').get()\n data = json.loads(json_text)\n \n # Navigate to product data\n product = data['props']['pageProps']['product']\n \n print({\n 'id': product['productId'],\n 'name': product['name'],\n 'price': product['pricing']['price'],\n 'brand': product['brand'],\n 'seller': product['seller']['username'],\n 'condition': product['condition'],\n 'images': [img['url'] for img in product['images']]\n })"
},
"sitemap_discovery": {
"url": "https://us.vestiairecollective.com/sitemaps/https_sitemap-en.xml",
"sub_sitemaps": "Organized by category, up to 50,000 listings each",
"selector": "url > loc::text"
},
"special_considerations": [
"Cloudflare protection requires HTTP/2 support",
"Residential proxies mandatory - datacenter IPs blocked",
"308 redirect = product unavailable/sold",
"Products change status frequently (sold items)",
"Best to scrape via sitemap discovery first",
"Recommended: ScrapFly SDK with asp=True parameter"
]
},
{
"site": "Farfetch",
"url": "https://www.farfetch.com/shopping/women/bags-1/items.aspx?designers=CHANEL",
"citation": "[5] Multiple Sources. 'Farfetch Scraping Methods.' Apify, Retailed.io, 2024.",
"platform": "Custom JavaScript application",
"anti_bot_protection": "Akamai Bot Manager + ETIMEDOUT errors",
"rendering_method": "JavaScript-rendered with possible GraphQL backend",
"data_extraction_method": "UNDOCUMENTED API OR GRAPHQL",
"recommended_approach": "Intercept XHR/Fetch requests to find internal API endpoints",
"blocking_status": "CONNECTION TIMEOUT (advanced protection)",
"correct_selectors": {
"note": "Cannot access site - timeout indicates aggressive rate limiting",
"investigation_method": "Use browser DevTools Network tab to find API endpoints",
"likely_api_pattern": "GraphQL query or REST API at /api/products or similar",
"alternative": "Use commercial Farfetch scraping API"
},
"implementation_code": {
"language": "javascript",
"code": "// RECOMMENDED: Intercept network requests to find API\nconst puppeteer = require('puppeteer');\n\nconst browser = await puppeteer.launch({ headless: false });\nconst page = await browser.newPage();\n\n// Listen to all requests\nawait page.setRequestInterception(true);\npage.on('request', request => {\n const url = request.url();\n if (url.includes('api') || url.includes('graphql') || url.includes('product')) {\n console.log('API REQUEST:', request.method(), url);\n console.log('POST DATA:', request.postData());\n }\n request.continue();\n});\n\n// Listen to responses\npage.on('response', async response => {\n const url = response.url();\n if (url.includes('product') && response.headers()['content-type']?.includes('json')) {\n console.log('API RESPONSE:', url);\n try {\n const data = await response.json();\n console.log('PRODUCT DATA:', JSON.stringify(data, null, 2));\n } catch (e) {}\n }\n});\n\nawait page.goto('https://www.farfetch.com/shopping/women/bags-1/items.aspx?designers=CHANEL');\n\n// Once you find the API endpoint, use it directly:\n// const response = await fetch('https://www.farfetch.com/api/products?...');"
},
"commercial_alternatives": [
{
"service": "Retailed.io",
"features": "Farfetch API with JSON/CSV output, 50 free requests",
"url": "https://www.retailed.io/datasources/api/farfetch-product"
},
{
"service": "Apify Farfetch Scraper",
"features": "Extract product catalog, prices, details from category pages",
"url": "https://apify.com/autofacts/farfetch"
}
],
"special_considerations": [
"Akamai Bot Manager is military-grade protection",
"Timeout errors indicate IP-based blocking or extreme rate limiting",
"Likely uses fingerprint-based detection",
"May require browser automation with advanced evasion",
"Consider using official partner APIs or commercial scrapers",
"Success rate with DIY scraping: <20%"
]
}
],
"technical_insights": {
"common_patterns": [
"All luxury sites use Next.js or React with client-side rendering",
"Product data embedded in __NEXT_DATA__ script tags (Fashionphile, Vestiaire)",
"Shopify sites expose /products.json API endpoint (Rebag)",
"Cloudflare or Akamai protection on ALL sites",
"JavaScript rendering requires waiting 3-10 seconds after page load",
"Lazy loading activated by scrolling"
],
"best_practices": [
"ALWAYS check for __NEXT_DATA__ script tag before scraping HTML",
"Use puppeteer-real-browser instead of regular puppeteer",
"Implement residential proxy rotation (not datacenter)",
"Add random delays 5-15 seconds between requests",
"Use realistic browser fingerprints (screen size, timezone, plugins)",
"Monitor for Cloudflare challenges and solve automatically",
"Prefer API endpoints over HTML scraping when available",
"Use HTTP/2 client libraries (httpx in Python, got in Node.js)"
],
"pitfalls": [
"Traditional Puppeteer gets detected immediately",
"Datacenter proxy IPs are blacklisted on luxury sites",
"Scraping too fast triggers rate limits (429 errors)",
"Not waiting for JavaScript rendering returns empty pages",
"Missing User-Agent or wrong browser signatures = 403 errors",
"Ignoring Cloudflare redirects breaks the session",
"Not handling 308 redirects on Vestiaire (sold items)"
],
"emerging_trends": [
"Cloudflare Turnstile replacing traditional CAPTCHAs",
"Per-customer ML-based bot detection (Cloudflare Enterprise)",
"Browser fingerprinting becoming more sophisticated",
"TLS fingerprint analysis to detect automation",
"Shift toward GraphQL APIs instead of REST"
]
},
"implementation_recommendations": [
{
"scenario": "Scraping Fashionphile or Vestiaire Collective",
"recommended_solution": "Parse __NEXT_DATA__ JSON with anti-bot proxy service",
"rationale": "10x faster than rendering JavaScript, data is pre-structured, lower detection risk",
"tools": "httpx (Python) or axios (Node.js) + residential proxies + ScrapFly",
"success_rate": "85-90%",
"code_example": "See Fashionphile implementation above"
},
{
"scenario": "Scraping Rebag (Shopify store)",
"recommended_solution": "Use /products.json public API endpoint",
"rationale": "Official Shopify feature, no HTML parsing needed, structured data",
"tools": "Simple HTTP client with rate limiting",
"success_rate": "95%+",
"code_example": "See Rebag implementation above"
},
{
"scenario": "Scraping The RealReal (Heavy Cloudflare)",
"recommended_solution": "puppeteer-real-browser with residential proxies",
"rationale": "Auto-solves Turnstile challenges, mimics real browser behavior",
"tools": "puppeteer-real-browser + rotating residential proxies",
"success_rate": "60-70%",
"code_example": "See The RealReal implementation above",
"alternative": "Use commercial API (BrightData Scraping Browser)"
},
{
"scenario": "Scraping Farfetch (Akamai protection)",
"recommended_solution": "Use commercial scraping API or intercept internal API",
"rationale": "Akamai is too advanced for DIY scraping, API intercept most reliable",
"tools": "Apify Farfetch Scraper or Retailed.io API",
"success_rate_diy": "15-20%",
"success_rate_commercial": "90%+",
"code_example": "See Farfetch network interception code above"
},
{
"scenario": "Large-scale scraping (1000+ products/day)",
"recommended_solution": "Commercial scraping infrastructure",
"rationale": "Residential proxies + CAPTCHA solving + maintenance costs exceed API fees",
"tools": "ScrapFly, BrightData, Oxylabs",
"monthly_cost": "$50-500 depending on volume",
"alternative": "Self-hosted with puppeteer-real-browser + proxy pool"
}
],
"community_insights": {
"popular_solutions": [
"puppeteer-real-browser (most mentioned for Cloudflare bypass)",
"ScrapFly SDK for managed anti-bot bypass",
"Shopify /products.json API for Shopify stores",
"__NEXT_DATA__ extraction for Next.js sites"
],
"controversial_topics": [
"Ethics of scraping luxury sites (terms of service violations)",
"Reliability of puppeteer-extra-stealth (declining effectiveness in 2025)",
"Cost/benefit of DIY vs commercial scraping services",
"Legal risks of aggressive scraping techniques"
],
"expert_opinions": [
"ScrapeOps: 'puppeteer-real-browser is the top choice for Cloudflare bypass in 2025'",
"Scrapfly: 'Parsing __NEXT_DATA__ is 10x faster and more reliable than DOM scraping'",
"BrightData: 'Luxury sites use per-customer ML models, generic bypass techniques <50% success'",
"ZenRows: 'Cloudflare Enterprise Bot Management detects puppeteer-extra-stealth immediately'"
]
},
"required_dependencies": {
"basic_scraping": [
"puppeteer-real-browser (npm install puppeteer-real-browser)",
"puppeteer-extra + puppeteer-extra-plugin-stealth",
"cheerio (HTML parsing)",
"axios or httpx (HTTP client with HTTP/2)"
],
"anti_bot_bypass": [
"xvfb (Linux virtual display for headless mode)",
"residential proxy service (Bright Data, Oxylabs, Smartproxy)",
"ScrapFly SDK (optional, managed solution)",
"2captcha or similar CAPTCHA solving service"
],
"python_alternative": [
"httpx (async HTTP with HTTP/2)",
"parsel (CSS selector parsing)",
"playwright-python (browser automation)",
"cloudscraper (Cloudflare bypass)"
]
},
"puppeteer_configuration": {
"basic_stealth": {
"install": "npm i puppeteer-extra puppeteer-extra-plugin-stealth",
"code": "const puppeteer = require('puppeteer-extra');\nconst StealthPlugin = require('puppeteer-extra-plugin-stealth');\npuppeteer.use(StealthPlugin());\n\nconst browser = await puppeteer.launch({\n headless: 'new',\n args: [\n '--no-sandbox',\n '--disable-setuid-sandbox',\n '--disable-blink-features=AutomationControlled'\n ]\n});"
},
"advanced_real_browser": {
"install": "npm i puppeteer-real-browser && sudo apt-get install xvfb",
"code": "const { connect } = require('puppeteer-real-browser');\n\nconst { page, browser } = await connect({\n headless: 'auto',\n fingerprint: true,\n turnstile: true,\n tf: true,\n args: ['--no-sandbox', '--disable-setuid-sandbox']\n});\n\nawait page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');\nawait page.setViewport({ width: 1920, height: 1080 });"
},
"with_residential_proxy": {
"code": "const browser = await puppeteer.launch({\n headless: true,\n args: [\n '--proxy-server=http://residential-proxy.example.com:8181',\n '--no-sandbox'\n ]\n});\n\nconst page = await browser.newPage();\nawait page.authenticate({\n username: 'proxy_username',\n password: 'proxy_password'\n});"
},
"recommended_headers": {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": "en-US,en;q=0.9",
"Accept-Encoding": "gzip, deflate, br",
"Connection": "keep-alive",
"Upgrade-Insecure-Requests": "1",
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none"
}
},
"testing_checklist": [
"Verify puppeteer-real-browser can solve Cloudflare Turnstile",
"Test __NEXT_DATA__ extraction on Fashionphile and Vestiaire",
"Confirm Rebag /products.json endpoint is accessible",
"Check if residential proxies are needed (test with/without)",
"Measure success rate over 100 requests",
"Monitor for 403, 429, or timeout errors",
"Verify product data completeness (title, price, image, URL)",
"Test pagination handling",
"Confirm proper error handling for sold/unavailable items"
],
"cost_analysis": {
"diy_scraping_monthly": {
"residential_proxies": "$50-200 (Bright Data, Smartproxy)",
"captcha_solving": "$10-50 (2captcha, if needed)",
"server_hosting": "$10-50 (VPS for running scrapers)",
"maintenance_time": "10-20 hours/month debugging blocks",
"total": "$70-300/month + significant time investment"
},
"commercial_api_monthly": {
"scrapfly_basic": "$50-100 (50k-100k requests)",
"brightdata_scraping_browser": "$100-500 (based on bandwidth)",
"apify_farfetch_scraper": "$49+ (based on usage)",
"retailed_io": "$0 (50 free requests) + paid tiers",
"total": "$50-500/month, zero maintenance",
"recommendation": "Commercial API more cost-effective for 10k+ products/month"
}
},
"final_recommendations": {
"immediate_fixes": [
"Install puppeteer-real-browser: npm install puppeteer-real-browser",
"Add waitUntil: 'networkidle2' and 5+ second delays",
"Check for __NEXT_DATA__ script tag before using CSS selectors",
"Use /products.json for Rebag instead of HTML scraping",
"Add residential proxy rotation to avoid IP blocks"
],
"short_term_strategy": [
"Implement __NEXT_DATA__ extraction for Fashionphile and Vestiaire",
"Use Shopify JSON API for Rebag",
"Deploy puppeteer-real-browser for The RealReal",
"Consider commercial API for Farfetch (or skip it)",
"Add proper error handling and retry logic"
],
"long_term_strategy": [
"Evaluate cost of DIY vs commercial APIs after 1 month",
"Consider ScrapFly or BrightData for managed solution",
"Implement sitemap scraping for Vestiaire (more reliable)",
"Monitor success rates and adjust approach per site",
"Stay updated on anti-bot technology changes"
]
}
}