← back to Watches
utils/price-scraper.js
296 lines
/**
* Watch Price Scraper using Playwright
* Fetches live pricing data from multiple watch marketplaces
*/
import { chromium } from 'playwright';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
class PriceScraper {
constructor() {
this.browser = null;
this.context = null;
this.resultsDir = path.join(__dirname, '../data/price-updates');
if (!fs.existsSync(this.resultsDir)) {
fs.mkdirSync(this.resultsDir, { recursive: true });
}
}
/**
* Initialize browser with stealth settings
*/
async init() {
if (!this.browser) {
this.browser = await chromium.launch({
headless: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-blink-features=AutomationControlled'
]
});
this.context = await this.browser.newContext({
userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
viewport: { width: 1920, height: 1080 },
locale: 'en-US'
});
}
return this.browser;
}
/**
* Scrape Chrono24 for watch price
*/
async scrapeChrono24(reference) {
await this.init();
const page = await this.context.newPage();
try {
const cleanRef = reference.replace(/[\s\.]/g, '');
const searchUrl = `https://www.chrono24.com/search/index.htm?query=omega+${cleanRef}`;
console.log(`🔍 Searching Chrono24 for ${reference}...`);
await page.goto(searchUrl, { waitUntil: 'networkidle', timeout: 30000 });
// Wait for results
await page.waitForSelector('.article-item-container, .js-article-item', { timeout: 10000 });
// Extract price data
const priceData = await page.evaluate(() => {
const items = document.querySelectorAll('.article-item-container, .js-article-item');
const prices = [];
items.forEach((item, index) => {
if (index >= 5) return; // Top 5 results
const priceElement = item.querySelector('.price, [data-testid="price"]');
const titleElement = item.querySelector('.text-bold, .article-title, h2');
const conditionElement = item.querySelector('.text-gray, .condition');
if (priceElement && titleElement) {
const priceText = priceElement.textContent.trim();
const priceMatch = priceText.match(/[\d,]+/);
const price = priceMatch ? parseInt(priceMatch[0].replace(/,/g, '')) : null;
prices.push({
title: titleElement.textContent.trim(),
price: price,
condition: conditionElement ? conditionElement.textContent.trim() : 'Unknown',
source: 'Chrono24'
});
}
});
return prices;
});
console.log(`✓ Found ${priceData.length} listings on Chrono24`);
return priceData;
} catch (error) {
console.error(`✗ Chrono24 scraping failed:`, error.message);
return [];
} finally {
await page.close();
}
}
/**
* Scrape WatchCharts for price index
*/
async scrapeWatchCharts(model, reference) {
await this.init();
const page = await this.context.newPage();
try {
const cleanRef = reference.replace(/[\s\.]/g, '');
const modelSlug = model.toLowerCase().replace(/[^a-z0-9]+/g, '-');
const url = `https://watchcharts.com/watches/${modelSlug}`;
console.log(`🔍 Checking WatchCharts for ${model}...`);
await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 });
// Extract market index data
const indexData = await page.evaluate(() => {
const priceElement = document.querySelector('.market-price, .price-display, .current-value');
const changeElement = document.querySelector('.price-change, .percentage-change');
return {
currentIndex: priceElement ? priceElement.textContent.trim() : null,
change: changeElement ? changeElement.textContent.trim() : null,
source: 'WatchCharts'
};
});
console.log(`✓ WatchCharts data retrieved`);
return indexData;
} catch (error) {
console.error(`✗ WatchCharts scraping failed:`, error.message);
return null;
} finally {
await page.close();
}
}
/**
* Scrape Bob's Watches
*/
async scrapeBobsWatches(reference) {
await this.init();
const page = await this.context.newPage();
try {
const cleanRef = reference.replace(/[\s\.]/g, '-').toLowerCase();
const searchUrl = `https://www.bobswatches.com/omega-${cleanRef}`;
console.log(`🔍 Searching Bob's Watches for ${reference}...`);
await page.goto(searchUrl, { waitUntil: 'networkidle', timeout: 30000 });
const priceData = await page.evaluate(() => {
const priceElement = document.querySelector('.product-price, .price, [itemprop="price"]');
const titleElement = document.querySelector('.product-title, h1');
const conditionElement = document.querySelector('.condition, .product-condition');
if (priceElement) {
const priceText = priceElement.textContent.trim();
const priceMatch = priceText.match(/[\d,]+/);
const price = priceMatch ? parseInt(priceMatch[0].replace(/,/g, '')) : null;
return {
title: titleElement ? titleElement.textContent.trim() : 'Unknown',
price: price,
condition: conditionElement ? conditionElement.textContent.trim() : 'Pre-owned',
source: "Bob's Watches"
};
}
return null;
});
if (priceData) {
console.log(`✓ Found listing on Bob's Watches`);
}
return priceData;
} catch (error) {
console.error(`✗ Bob's Watches scraping failed:`, error.message);
return null;
} finally {
await page.close();
}
}
/**
* Scrape all sources for a watch
*/
async scrapeAllSources(watch) {
console.log(`\n📊 Scraping prices for ${watch.model} (${watch.reference})...`);
const results = {
watch: {
id: watch.id,
model: watch.model,
reference: watch.reference,
series: watch.series
},
timestamp: new Date().toISOString(),
prices: []
};
// Scrape Chrono24
const chrono24Prices = await this.scrapeChrono24(watch.reference);
results.prices.push(...chrono24Prices);
// Add delay to avoid rate limiting
await new Promise(resolve => setTimeout(resolve, 2000));
// Scrape WatchCharts
const watchChartsData = await this.scrapeWatchCharts(watch.model, watch.reference);
if (watchChartsData) {
results.prices.push(watchChartsData);
}
await new Promise(resolve => setTimeout(resolve, 2000));
// Scrape Bob's Watches
const bobsData = await this.scrapeBobsWatches(watch.reference);
if (bobsData) {
results.prices.push(bobsData);
}
// Calculate average market price
const validPrices = results.prices.filter(p => p.price).map(p => p.price);
if (validPrices.length > 0) {
results.averageMarketPrice = Math.round(validPrices.reduce((a, b) => a + b, 0) / validPrices.length);
results.lowestPrice = Math.min(...validPrices);
results.highestPrice = Math.max(...validPrices);
}
console.log(`✓ Scraped ${results.prices.length} price points`);
if (results.averageMarketPrice) {
console.log(` Average: $${results.averageMarketPrice.toLocaleString()}`);
console.log(` Range: $${results.lowestPrice.toLocaleString()} - $${results.highestPrice.toLocaleString()}`);
}
return results;
}
/**
* Save scraping results
*/
saveResults(results) {
const filename = `price-update-${results.watch.id}-${new Date().toISOString().split('T')[0]}.json`;
const filepath = path.join(this.resultsDir, filename);
fs.writeFileSync(filepath, JSON.stringify(results, null, 2));
console.log(`💾 Results saved: ${filepath}`);
return filepath;
}
/**
* Close browser
*/
async close() {
if (this.browser) {
await this.browser.close();
this.browser = null;
this.context = null;
}
}
}
export default PriceScraper;
// CLI usage
if (import.meta.url === `file://${process.argv[1]}`) {
const scraper = new PriceScraper();
(async () => {
try {
// Load watches database
const dbPath = path.join(__dirname, '../data/watches.json');
const database = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
// Scrape first watch as example
const watch = database.watches[0];
const results = await scraper.scrapeAllSources(watch);
scraper.saveResults(results);
} catch (error) {
console.error('Error:', error);
process.exit(1);
} finally {
await scraper.close();
}
})();
}