`scripts/daily-full-crawl.js`

114 lines
#!/usr/bin/env node

/**
 * DAILY FULL WINE CRAWL - 6AM
 * Comprehensive crawl of ALL wine sources
 * Includes old and new wines for price tracking
 */

const axios = require('axios');
const fs = require('fs').promises;
const path = require('path');

const DATA_DIR = '/root/Projects/wine-finder-next/data';
const DAILY_CRAWL_FILE = path.join(DATA_DIR, `daily-crawl-${new Date().toISOString().split('T')[0]}.json`);

// Comprehensive search list - 100+ searches
const COMPREHENSIVE_SEARCHES = [
  // Red Varietals
  'cabernet sauvignon', 'pinot noir', 'merlot', 'syrah', 'shiraz', 'zinfandel',
  'malbec', 'tempranillo', 'grenache', 'sangiovese', 'nebbiolo', 'barbera',
  'petite sirah', 'carmenere', 'cabernet franc', 'gamay', 'mourvedre',

  // White Varietals
  'chardonnay', 'sauvignon blanc', 'pinot grigio', 'pinot gris', 'riesling',
  'viognier', 'gewurztraminer', 'moscato', 'chenin blanc', 'albarino',
  'gruner veltliner', 'torrontes', 'semillon', 'marsanne', 'roussanne',

  // Blends
  'red blend', 'white blend', 'bordeaux blend', 'rhone blend', 'meritage',
  'super tuscan', 'gst blend', 'rose blend',

  // Famous Regions
  'napa valley', 'sonoma', 'paso robles', 'willamette valley', 'santa barbara',
  'russian river', 'carneros', 'howell mountain', 'rutherford', 'oakville',
  'bordeaux', 'burgundy', 'tuscany', 'rioja', 'barolo', 'chianti', 'priorat',
  'champagne', 'prosecco', 'cava', 'porto', 'sherry', 'madeira',

  // Premium Wineries
  'caymus', 'silver oak', 'opus one', 'stags leap', 'jordan', 'duckhorn',
  'shafer', 'screaming eagle', 'harlan', 'dominus', 'ridge', 'paul hobbs',
  'kistler', 'penfolds', 'cloudy bay', 'domaine de la romanee conti',
  'chateau margaux', 'chateau lafite', 'sassicaia', 'antinori', 'gaja',

  // Price Tiers
  'under 15', 'under 20', 'under 30', 'under 50', 'under 100',
  '50 to 100', '100 to 200', '200 to 500', 'over 500',

  // Special Categories
  'premium wine', 'luxury wine', 'reserve wine', 'estate wine',
  'old vine', 'single vineyard', 'organic wine', 'biodynamic wine',
  'natural wine', 'orange wine', 'new release', 'library wine',
  'cult wine', 'allocated wine', 'limited production'
];

async function crawlAllWines() {
  console.log('========================================');
  console.log('DAILY FULL WINE CRAWL');
  console.log('Started:', new Date().toISOString());
  console.log(`Total searches: ${COMPREHENSIVE_SEARCHES.length}`);
  console.log('========================================\n');

  const allWines = [];
  let searchCount = 0;

  for (const query of COMPREHENSIVE_SEARCHES) {
    searchCount++;
    console.log(`[${searchCount}/${COMPREHENSIVE_SEARCHES.length}] ${query}`);

    try {
      const response = await axios.get('http://localhost:7201/api/search', {
        params: {
          q: query,
          sources: 'vivino,totalwine,kl',
          limit: 50
        },
        timeout: 30000
      });

      const wines = response.data.wines || [];
      allWines.push(...wines);
      console.log(`  Found: ${wines.length} wines`);

    } catch (error) {
      console.log(`  Error: ${error.message}`);
    }

    // Rate limiting
    await new Promise(resolve => setTimeout(resolve, 3000));
  }

  // Deduplicate
  const uniqueWines = Array.from(
    new Map(allWines.map(w => [`${w.name}|${w.source}|${w.vintage}`, w])).values()
  );

  console.log(`\nTotal wines: ${allWines.length}`);
  console.log(`Unique wines: ${uniqueWines.length}`);

  // Save results
  await fs.mkdir(DATA_DIR, { recursive: true });
  await fs.writeFile(DAILY_CRAWL_FILE, JSON.stringify({
    timestamp: new Date().toISOString(),
    searchCount: COMPREHENSIVE_SEARCHES.length,
    totalWines: allWines.length,
    uniqueWines: uniqueWines.length,
    wines: uniqueWines
  }, null, 2));

  console.log(`\n✓ Saved to: ${DAILY_CRAWL_FILE}`);
  console.log('========================================');
}

crawlAllWines().catch(console.error);