← back to Handbag Authentication
scripts/daily-price-crawler.js
462 lines
#!/usr/bin/env node
/**
* Daily Price Crawler for Luxury Retailers
* Crawls TheRealReal, Fashionphile, Rebag, and Vestiaire Collective
* Tracks price history for luxury bags to build comprehensive price database
* Runs daily at 6 AM via cron
*/
require('dotenv').config({ path: '/root/Projects/handbag-authentication/.env' });
const sqlite3 = require('sqlite3').verbose();
const axios = require('axios');
const cheerio = require('cheerio');
const fs = require('fs');
const path = require('path');
// Database path
const DB_PATH = '/root/Projects/handbag-authentication/data/handbags.db';
// Luxury brands to track
const LUXURY_BRANDS = [
'hermes', 'chanel', 'louis-vuitton', 'gucci', 'dior',
'prada', 'fendi', 'bottega-veneta', 'celine', 'saint-laurent',
'balenciaga', 'valentino', 'givenchy', 'loewe', 'chloe'
];
// Retailers to crawl
const RETAILERS = {
therealreal: {
name: 'TheRealReal',
baseUrl: 'https://www.therealreal.com',
searchUrl: 'https://www.therealreal.com/designers/{brand}/handbags',
selector: {
items: '.product-card',
title: '.product-card__description',
price: '.product-card__price',
link: 'a.product-card__link',
image: '.product-card__image img'
}
},
fashionphile: {
name: 'Fashionphile',
baseUrl: 'https://www.fashionphile.com',
searchUrl: 'https://www.fashionphile.com/shop/{brand}',
selector: {
items: '.product-item',
title: '.product-name',
price: '.price',
link: 'a.product-link',
image: '.product-image img'
}
},
rebag: {
name: 'Rebag',
baseUrl: 'https://www.rebag.com',
searchUrl: 'https://www.rebag.com/shop?q={brand}&category=bags',
selector: {
items: '.product-card',
title: '.product-title',
price: '.product-price',
link: 'a.product-link',
image: '.product-image img'
}
},
vestiaire: {
name: 'Vestiaire Collective',
baseUrl: 'https://www.vestiairecollective.com',
searchUrl: 'https://www.vestiairecollective.com/search/?q={brand}&category=bags',
selector: {
items: '.product-item',
title: '.product-title',
price: '.product-price',
link: 'a.product-link',
image: '.product-image img'
}
}
};
class PriceCrawler {
constructor() {
this.db = new sqlite3.Database(DB_PATH);
this.initDatabase();
this.crawlStats = {
totalItems: 0,
newItems: 0,
updatedPrices: 0,
errors: []
};
}
initDatabase() {
// Create price history table if it doesn't exist
this.db.run(`
CREATE TABLE IF NOT EXISTS price_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
retailer TEXT NOT NULL,
brand TEXT NOT NULL,
item_name TEXT NOT NULL,
item_detail TEXT,
model TEXT,
condition TEXT,
price_usd REAL NOT NULL,
currency TEXT DEFAULT 'USD',
product_url TEXT,
image_url TEXT,
external_id TEXT,
crawl_date DATE NOT NULL,
first_seen_date DATE,
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
days_on_market INTEGER DEFAULT 0,
price_change REAL DEFAULT 0,
created_at TEXT DEFAULT CURRENT_TIMESTAMP,
UNIQUE(retailer, external_id, crawl_date)
)
`);
// Create indexes for better query performance
this.db.run(`CREATE INDEX IF NOT EXISTS idx_price_history_date ON price_history(crawl_date)`);
this.db.run(`CREATE INDEX IF NOT EXISTS idx_price_history_brand ON price_history(brand)`);
this.db.run(`CREATE INDEX IF NOT EXISTS idx_price_history_retailer ON price_history(retailer)`);
this.db.run(`CREATE INDEX IF NOT EXISTS idx_price_history_model ON price_history(model)`);
// Create price analytics table
this.db.run(`
CREATE TABLE IF NOT EXISTS price_analytics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
retailer TEXT NOT NULL,
brand TEXT NOT NULL,
model TEXT,
avg_price REAL,
min_price REAL,
max_price REAL,
price_trend TEXT,
last_updated TEXT DEFAULT CURRENT_TIMESTAMP
)
`);
}
async crawlRetailer(retailerKey, retailer) {
console.log(`\n📍 Crawling ${retailer.name}...`);
for (const brand of LUXURY_BRANDS) {
try {
const url = retailer.searchUrl.replace('{brand}', brand);
console.log(` 🔍 Searching ${brand} bags on ${retailer.name}...`);
// Add delay to avoid rate limiting
await this.delay(2000 + Math.random() * 3000);
const response = await axios.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1'
},
timeout: 30000
});
const $ = cheerio.load(response.data);
const items = $(retailer.selector.items);
console.log(` ✓ Found ${items.length} ${brand} items`);
items.each((index, element) => {
if (index >= 50) return; // Limit to 50 items per brand to avoid overwhelming
try {
const $item = $(element);
const title = $item.find(retailer.selector.title).text().trim();
const priceText = $item.find(retailer.selector.price).text().trim();
const link = $item.find(retailer.selector.link).attr('href');
const image = $item.find(retailer.selector.image).attr('src') || $item.find(retailer.selector.image).attr('data-src');
// Parse price
const price = this.parsePrice(priceText);
if (!price || price <= 0) return;
// Generate external ID
const externalId = this.extractIdFromUrl(link) || `${retailerKey}_${brand}_${Date.now()}_${index}`;
// Extract model and condition from title if possible
const { model, condition } = this.extractDetails(title);
// Prepare data for insertion
const data = {
retailer: retailer.name,
brand: this.normalizeBrand(brand),
item_name: title,
item_detail: title,
model: model,
condition: condition,
price_usd: price,
product_url: link?.startsWith('http') ? link : `${retailer.baseUrl}${link}`,
image_url: image,
external_id: externalId,
crawl_date: new Date().toISOString().split('T')[0]
};
this.saveToDatabase(data);
this.crawlStats.totalItems++;
} catch (itemError) {
console.error(` ❌ Error processing item: ${itemError.message}`);
}
});
} catch (error) {
console.error(` ❌ Error crawling ${brand} on ${retailer.name}: ${error.message}`);
this.crawlStats.errors.push(`${retailer.name} - ${brand}: ${error.message}`);
}
}
}
parsePrice(priceText) {
// Remove currency symbols and extract number
const cleaned = priceText.replace(/[^0-9.,]/g, '');
const price = parseFloat(cleaned.replace(',', ''));
return isNaN(price) ? null : price;
}
extractIdFromUrl(url) {
if (!url) return null;
// Try to extract product ID from URL
const matches = url.match(/\/([0-9]+)(?:\/|$)/);
return matches ? matches[1] : null;
}
normalizeBrand(brand) {
const brandMap = {
'hermes': 'Hermes',
'chanel': 'Chanel',
'louis-vuitton': 'Louis Vuitton',
'gucci': 'Gucci',
'dior': 'Dior',
'prada': 'Prada',
'fendi': 'Fendi',
'bottega-veneta': 'Bottega Veneta',
'celine': 'Celine',
'saint-laurent': 'Saint Laurent',
'balenciaga': 'Balenciaga',
'valentino': 'Valentino',
'givenchy': 'Givenchy',
'loewe': 'Loewe',
'chloe': 'Chloe'
};
return brandMap[brand] || brand;
}
extractDetails(title) {
const models = {
// Hermes
'birkin': 'Birkin', 'kelly': 'Kelly', 'constance': 'Constance',
'garden party': 'Garden Party', 'evelyne': 'Evelyne', 'bolide': 'Bolide',
// Chanel
'classic flap': 'Classic Flap', 'boy': 'Boy', '2.55': '2.55',
'19': '19', 'gabrielle': 'Gabrielle',
// Louis Vuitton
'neverfull': 'Neverfull', 'speedy': 'Speedy', 'alma': 'Alma',
'pochette': 'Pochette', 'capucines': 'Capucines',
// Dior
'lady dior': 'Lady Dior', 'saddle': 'Saddle', 'book tote': 'Book Tote',
// Gucci
'marmont': 'Marmont', 'dionysus': 'Dionysus', 'jackie': 'Jackie'
};
const conditions = {
'new': 'New', 'like new': 'Like New', 'excellent': 'Excellent',
'very good': 'Very Good', 'good': 'Good', 'fair': 'Fair'
};
let model = null;
let condition = null;
const lowerTitle = title.toLowerCase();
// Find model
for (const [key, value] of Object.entries(models)) {
if (lowerTitle.includes(key)) {
model = value;
break;
}
}
// Find condition
for (const [key, value] of Object.entries(conditions)) {
if (lowerTitle.includes(key)) {
condition = value;
break;
}
}
return { model, condition };
}
saveToDatabase(data) {
// First check if this item already exists to get first_seen_date
const checkQuery = `
SELECT first_seen_date, price_usd, MIN(crawl_date) as first_date
FROM price_history
WHERE retailer = ? AND external_id = ?
GROUP BY retailer, external_id
LIMIT 1
`;
this.db.get(checkQuery, [data.retailer, data.external_id], (err, existing) => {
let firstSeenDate = data.crawl_date;
let daysOnMarket = 0;
let priceChange = 0;
if (!err && existing) {
// Item exists, use the original first_seen_date
firstSeenDate = existing.first_seen_date || existing.first_date || data.crawl_date;
// Calculate days on market
const firstDate = new Date(firstSeenDate);
const currentDate = new Date(data.crawl_date);
daysOnMarket = Math.floor((currentDate - firstDate) / (1000 * 60 * 60 * 24));
// Calculate price change
if (existing.price_usd) {
priceChange = data.price_usd - existing.price_usd;
}
}
const insertQuery = `
INSERT OR REPLACE INTO price_history (
retailer, brand, item_name, item_detail, model, condition,
price_usd, product_url, image_url, external_id, crawl_date,
first_seen_date, days_on_market, price_change, last_updated
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))
`;
this.db.run(insertQuery, [
data.retailer,
data.brand,
data.item_name,
data.item_detail,
data.model,
data.condition,
data.price_usd,
data.product_url,
data.image_url,
data.external_id,
data.crawl_date,
firstSeenDate,
daysOnMarket,
priceChange
], (insertErr) => {
if (insertErr) {
console.error(` ❌ Database error: ${insertErr.message}`);
} else {
const now = new Date().toISOString();
if (!existing) {
this.crawlStats.newItems++;
console.log(` ✨ NEW: ${data.item_name} - First seen: ${firstSeenDate} | Last updated: ${now}`);
} else if (priceChange !== 0) {
this.crawlStats.updatedPrices++;
const changeSymbol = priceChange > 0 ? '📈' : '📉';
console.log(` ${changeSymbol} PRICE CHANGE: ${data.item_name} - $${priceChange.toFixed(2)} (${daysOnMarket} days on market) | Updated: ${now}`);
} else {
console.log(` ✓ CHECKED: ${data.item_name} - No change (${daysOnMarket} days) | Updated: ${now}`);
}
}
});
});
}
async delay(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
async updateAnalytics() {
console.log('\n📊 Updating price analytics...');
const query = `
INSERT OR REPLACE INTO price_analytics (retailer, brand, model, avg_price, min_price, max_price, price_trend)
SELECT
retailer,
brand,
model,
AVG(price_usd) as avg_price,
MIN(price_usd) as min_price,
MAX(price_usd) as max_price,
CASE
WHEN COUNT(DISTINCT crawl_date) > 1 THEN 'tracking'
ELSE 'new'
END as price_trend
FROM price_history
WHERE crawl_date >= date('now', '-30 days')
GROUP BY retailer, brand, model
`;
this.db.run(query, (err) => {
if (err) {
console.error('Error updating analytics:', err);
} else {
console.log('✅ Analytics updated successfully');
}
});
}
async crawlAll() {
console.log('🚀 Starting Daily Price Crawl');
console.log(`📅 Date: ${new Date().toISOString()}`);
console.log('🛍️ Retailers: TheRealReal, Fashionphile, Rebag, Vestiaire Collective');
console.log(`👜 Brands: ${LUXURY_BRANDS.join(', ')}\n`);
const startTime = Date.now();
for (const [key, retailer] of Object.entries(RETAILERS)) {
await this.crawlRetailer(key, retailer);
}
await this.updateAnalytics();
const duration = ((Date.now() - startTime) / 1000 / 60).toFixed(2);
// Log summary
console.log('\n' + '='.repeat(60));
console.log('📈 CRAWL SUMMARY');
console.log('='.repeat(60));
console.log(`⏱️ Duration: ${duration} minutes`);
console.log(`📦 Total items processed: ${this.crawlStats.totalItems}`);
console.log(`✅ New items added: ${this.crawlStats.newItems}`);
if (this.crawlStats.errors.length > 0) {
console.log(`\n⚠️ Errors encountered (${this.crawlStats.errors.length}):`);
this.crawlStats.errors.forEach(error => console.log(` - ${error}`));
}
console.log('\n✨ Daily price crawl completed successfully!');
// Log to file
const logEntry = {
date: new Date().toISOString(),
duration: duration,
totalItems: this.crawlStats.totalItems,
newItems: this.crawlStats.newItems,
errors: this.crawlStats.errors
};
const logFile = '/root/Projects/handbag-authentication/logs/price-crawl.log';
fs.appendFileSync(logFile, JSON.stringify(logEntry) + '\n');
// Close database connection
this.db.close();
}
}
// Run the crawler
if (require.main === module) {
const crawler = new PriceCrawler();
crawler.crawlAll().catch(error => {
console.error('Fatal error:', error);
process.exit(1);
});
}
module.exports = PriceCrawler;