← back to Wine Finder
crawler/wine-crawler.js
320 lines
#!/usr/bin/env node
/**
* Comprehensive Wine Crawler
*
* Crawls wine data from multiple sources across the US:
* - K&L Wine Merchants (California)
* - Total Wine & More (Nationwide)
* - Vivino (Global marketplace)
* - WineSensed (UC Davis dataset - 350k vintages)
* - X-Wines (100k wines dataset)
* - Wine Enthusiast (Review data)
* - UC Davis AVA (Geographic regions)
*
* Logs results to Google Sheets for historical tracking
*/
const path = require('path');
require('dotenv').config({ path: path.join(__dirname, '..', '.env') });
const wineAggregator = require('../services/wineAggregator');
const googleSheets = require('../services/googleSheets');
const NewItemsTracker = require('../services/newItemsTracker');
const fs = require('fs').promises;
// Popular wine search terms for broad coverage
const SEARCH_QUERIES = [
// Red wines - popular varietals
'Cabernet Sauvignon',
'Pinot Noir',
'Merlot',
'Zinfandel',
'Syrah',
'Malbec',
// White wines
'Chardonnay',
'Sauvignon Blanc',
'Pinot Grigio',
'Riesling',
// Popular regions
'Napa Valley',
'Sonoma',
'Bordeaux',
'Burgundy',
'Tuscany',
// Price categories
'under $20',
'under $30',
'premium wine',
// Special categories
'organic wine',
'natural wine',
'new release'
];
class WineCrawler {
constructor() {
this.startTime = new Date();
this.newItemsTracker = new NewItemsTracker();
this.results = {
totalWines: 0,
newWines: 0,
bySource: {},
errors: [],
queries: 0
};
}
log(message) {
const timestamp = new Date().toISOString();
console.log(`[${timestamp}] ${message}`);
}
async crawl() {
this.log('Starting comprehensive wine crawl...');
this.log(`Crawling ${SEARCH_QUERIES.length} search queries across all sources`);
const allWines = [];
for (let i = 0; i < SEARCH_QUERIES.length; i++) {
const query = SEARCH_QUERIES[i];
this.log(`\n[${i+1}/${SEARCH_QUERIES.length}] Searching for: "${query}"`);
try {
const result = await wineAggregator.searchAll(query, {
sources: ['kl', 'vivino', 'totalwine', 'winesensed', 'xwines', 'wineenthusiast', 'ucdavis-ava'],
deduplicate: true,
sortBy: 'price',
limit: 50 // Get top 50 per query
});
if (result.success && result.wines.length > 0) {
this.log(` ✓ Found ${result.wines.length} wines`);
// Track by source
if (result.sources) {
result.sources.forEach(source => {
if (!this.results.bySource[source.source]) {
this.results.bySource[source.source] = 0;
}
this.results.bySource[source.source] += source.count;
});
}
// Add search query to each wine for context
result.wines.forEach(wine => {
wine.searchQuery = query;
});
allWines.push(...result.wines);
this.results.totalWines += result.wines.length;
} else {
this.log(` ⚠ No wines found for "${query}"`);
}
if (result.errors && result.errors.length > 0) {
result.errors.forEach(err => {
this.log(` ⚠ ${err.source}: ${err.error}`);
this.results.errors.push({ query, ...err });
});
}
this.results.queries++;
// Rate limiting - wait 2 seconds between queries
if (i < SEARCH_QUERIES.length - 1) {
await new Promise(resolve => setTimeout(resolve, 2000));
}
} catch (error) {
this.log(` ✗ Error searching "${query}": ${error.message}`);
this.results.errors.push({ query, error: error.message });
}
}
// Deduplicate across all queries
this.log('\n\nDeduplicating wines across all queries...');
const uniqueWines = this.deduplicateAllWines(allWines);
this.log(`Total wines after deduplication: ${uniqueWines.length}`);
// Identify NEW wines
this.log('\n\nIdentifying NEW wines (not seen before)...');
const trackingResult = await this.newItemsTracker.processWines(uniqueWines);
const newWines = trackingResult.newWines;
this.results.newWines = newWines.length;
this.log(`\n🆕 NEW WINES: ${newWines.length} out of ${uniqueWines.length}`);
this.log(this.newItemsTracker.getNewItemsSummary());
// Save to Google Sheets (only NEW wines to reduce clutter)
if (newWines.length > 0) {
this.log(`\nLogging ${newWines.length} NEW wines to Google Sheets...`);
try {
await this.logToSheets(newWines);
this.log('✓ Successfully logged NEW wines to Google Sheets');
} catch (error) {
this.log(`✗ Error logging to Google Sheets: ${error.message}`);
this.results.errors.push({ step: 'google_sheets', error: error.message });
}
} else {
this.log('\n✓ No new wines to log to Google Sheets (all wines seen before)');
}
// Save to local JSON file as backup (all wines)
await this.saveBackup(uniqueWines);
// Save NEW wines to separate file for easy access
if (newWines.length > 0) {
await this.saveNewWinesReport(newWines);
}
// Print summary
this.printSummary(uniqueWines);
return { allWines: uniqueWines, newWines };
}
deduplicateAllWines(wines) {
const uniqueMap = new Map();
wines.forEach(wine => {
const key = `${wine.name}_${wine.vintage || ''}_${wine.source || ''}`
.toLowerCase()
.replace(/[^a-z0-9]/g, '');
if (!uniqueMap.has(key)) {
uniqueMap.set(key, wine);
} else {
// Keep wine with more complete data
const existing = uniqueMap.get(key);
if (this.isMoreComplete(wine, existing)) {
uniqueMap.set(key, wine);
}
}
});
return Array.from(uniqueMap.values());
}
isMoreComplete(wine1, wine2) {
let score1 = 0;
let score2 = 0;
if (wine1.price) score1++;
if (wine2.price) score2++;
if (wine1.rating) score1++;
if (wine2.rating) score2++;
if (wine1.url) score1++;
if (wine2.url) score2++;
if (wine1.vintage) score1++;
if (wine2.vintage) score2++;
return score1 > score2;
}
async logToSheets(wines) {
// Log in batches of 50 to avoid overwhelming the API
const batchSize = 50;
for (let i = 0; i < wines.length; i += batchSize) {
const batch = wines.slice(i, i + batchSize);
await googleSheets.logMultipleWines(batch);
// Wait between batches
if (i + batchSize < wines.length) {
await new Promise(resolve => setTimeout(resolve, 1000));
}
}
}
async saveBackup(wines) {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `/root/Projects/wine-finder/data/crawl-${timestamp}.json`;
try {
await fs.writeFile(filename, JSON.stringify({
timestamp: this.startTime.toISOString(),
totalWines: wines.length,
wines: wines,
results: this.results
}, null, 2));
this.log(`\n✓ Backup saved to: ${filename}`);
} catch (error) {
this.log(`✗ Error saving backup: ${error.message}`);
}
}
async saveNewWinesReport(newWines) {
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const filename = `/root/Projects/wine-finder/data/NEW-wines-${timestamp}.json`;
try {
await fs.writeFile(filename, JSON.stringify({
timestamp: this.startTime.toISOString(),
newWinesCount: newWines.length,
newWines: newWines,
summary: this.newItemsTracker.getNewItemsSummary()
}, null, 2));
this.log(`✓ NEW wines report saved to: ${filename}`);
} catch (error) {
this.log(`✗ Error saving new wines report: ${error.message}`);
}
}
printSummary(wines) {
const endTime = new Date();
const duration = ((endTime - this.startTime) / 1000 / 60).toFixed(2);
console.log('\n' + '='.repeat(60));
console.log('CRAWL SUMMARY');
console.log('='.repeat(60));
console.log(`Start Time: ${this.startTime.toISOString()}`);
console.log(`End Time: ${endTime.toISOString()}`);
console.log(`Duration: ${duration} minutes`);
console.log(`Queries Run: ${this.results.queries}`);
console.log(`Total Wines: ${wines.length}`);
console.log(`🆕 NEW Wines: ${this.results.newWines} (${wines.length > 0 ? ((this.results.newWines / wines.length) * 100).toFixed(1) : 0}%)`);
console.log();
console.log('Wines by Source:');
Object.keys(this.results.bySource).sort().forEach(source => {
console.log(` ${source.padEnd(20)} ${this.results.bySource[source]}`);
});
if (this.results.errors.length > 0) {
console.log();
console.log(`Errors: ${this.results.errors.length}`);
this.results.errors.slice(0, 10).forEach(err => {
console.log(` - ${err.source || err.query}: ${err.error}`);
});
}
console.log('='.repeat(60));
console.log();
}
}
// Run if executed directly
if (require.main === module) {
const crawler = new WineCrawler();
crawler.crawl()
.then(() => {
console.log('✓ Wine crawl completed successfully');
process.exit(0);
})
.catch(error => {
console.error('✗ Wine crawl failed:', error);
process.exit(1);
});
}
module.exports = WineCrawler;