`services/wineSensedScraper.js`

287 lines
const axios = require('axios');
const csv = require('csv-parser');
const { Readable } = require('stream');

/**
 * WineSensed Dataset Scraper
 *
 * Learning to Taste: A Multimodal Wine Dataset
 * Source: UC Davis / NeurIPS 2023 / DTU
 *
 * Dataset Details:
 * - 350,000+ unique wine vintages
 * - 824,000 wine reviews
 * - 897,000 wine label images
 * - Data collected from Vivino (May 2023)
 *
 * Fields: vintage_id, wine name, year, winery_id, country, region,
 *         price (USD), rating, grape composition, alcohol %, reviews
 *
 * Download URL: https://data.dtu.dk/articles/dataset/WineSensed_Learning_to_Taste_A_Multimodal_Wine_Dataset/23376560
 * License: CC BY-NC-ND 4.0
 */

class WineSensedScraper {
  constructor() {
    this.datasetUrl = 'https://data.dtu.dk/ndownloader/articles/23376560/versions/1';
    this.projectUrl = 'https://thoranna.github.io/learning_to_taste/';

    // Cache for dataset (in production, use Redis or file-based cache)
    this.dataCache = null;
    this.cacheTimestamp = null;
    this.cacheDuration = 24 * 60 * 60 * 1000; // 24 hours

    this.datasetInfo = {
      name: 'WineSensed: Learning to Taste',
      source: 'UC Davis / NeurIPS 2023',
      uniqueVintages: 350000,
      reviews: 824000,
      images: 897000,
      vintageRange: '1950-2021',
      dataCollection: 'May 2023',
      license: 'CC BY-NC-ND 4.0'
    };
  }

  /**
   * Search WineSensed dataset for wines
   * Note: This requires the CSV dataset to be downloaded locally
   */
  async search(query, options = {}) {
    try {
      const {
        limit = 50,
        minRating = 0,
        maxPrice = 10000,
        minYear = 1950,
        maxYear = 2023,
        country = null,
        region = null
      } = options;

      // Try to load sample data from JSON file
      const fs = require('fs');
      const path = require('path');
      const dataPath = path.join(__dirname, '../data/wine_sample_data.json');

      if (fs.existsSync(dataPath)) {
        const rawData = fs.readFileSync(dataPath, 'utf8');
        const wines = JSON.parse(rawData);

        // Filter by query
        const queryLower = query.toLowerCase();
        let filtered = wines.filter(wine => {
          const name = (wine.wine || '').toLowerCase();
          const winery = (wine.winery || '').toLowerCase();
          const region = (wine.region || '').toLowerCase();
          const grapes = (wine.grape_composition || '').toLowerCase();

          return name.includes(queryLower) ||
                 winery.includes(queryLower) ||
                 region.includes(queryLower) ||
                 grapes.includes(queryLower);
        });

        // Apply filters
        filtered = filtered.filter(wine => {
          const price = parseFloat(wine.price) || 0;
          const rating = parseFloat(wine.rating) || 0;
          const year = parseInt(wine.year) || 0;

          return rating >= minRating &&
                 price <= maxPrice &&
                 year >= minYear &&
                 year <= maxYear;
        });

        // Limit results
        filtered = filtered.slice(0, limit);

        // Format wines
        const formattedWines = filtered.map(wine => ({
          name: wine.wine || 'Unknown',
          vintage: wine.year || '',
          price: parseFloat(wine.price) || null,
          rating: parseFloat(wine.rating) || null,
          country: wine.country || '',
          region: wine.region || '',
          winery: wine.winery || '',
          grapes: wine.grape_composition || '',
          alcohol: wine.alcohol_percentage || '',
          ratingsCount: parseInt(wine.reviews) || 0,
          url: `https://www.vivino.com/search/wines?q=${encodeURIComponent(wine.wine)}`,
          availability: 'Academic Dataset (UC Davis)',
          source: 'UC Davis Wine Database',
          type: wine.grape_composition ? wine.grape_composition.split(',')[0].trim() : 'Wine',
          vintageId: wine.vintage_id || ''
        }));

        return {
          success: true,
          wines: formattedWines,
          count: formattedWines.length,
          source: 'UC Davis Wine Database',
          datasetInfo: '350k+ wines from UC Davis/NeurIPS academic research',
          timestamp: new Date().toISOString()
        };
      }

      // Fallback if no data file
      return {
        success: true,
        wines: [],
        count: 0,
        source: 'WineSensed (UC Davis/NeurIPS)',
        message: 'WineSensed dataset requires local download. Dataset contains 350k+ vintages with historical pricing.',
        datasetInfo: this.datasetInfo,
        timestamp: new Date().toISOString()
      };

    } catch (error) {
      console.error('WineSensed scraper error:', error.message);
      return {
        success: false,
        wines: [],
        error: error.message,
        source: 'WineSensed'
      };
    }
  }

  /**
   * Load and parse CSV data from local file
   * Requires dataset to be downloaded first
   */
  async loadDataset(csvPath) {
    return new Promise((resolve, reject) => {
      const results = [];
      const fs = require('fs');

      if (!fs.existsSync(csvPath)) {
        return reject(new Error(`Dataset file not found: ${csvPath}`));
      }

      fs.createReadStream(csvPath)
        .pipe(csv())
        .on('data', (data) => results.push(data))
        .on('end', () => {
          this.dataCache = results;
          this.cacheTimestamp = Date.now();
          resolve(results);
        })
        .on('error', reject);
    });
  }

  /**
   * Search loaded dataset
   */
  async searchLoadedData(query, options = {}) {
    if (!this.dataCache) {
      return {
        success: false,
        wines: [],
        error: 'Dataset not loaded. Call loadDataset() first.',
        source: 'WineSensed'
      };
    }

    const {
      limit = 50,
      minRating = 0,
      maxPrice = 10000
    } = options;

    const queryLower = query.toLowerCase();

    // Filter wines by query
    let wines = this.dataCache.filter(wine => {
      const name = (wine.wine || '').toLowerCase();
      const winery = (wine.winery || '').toLowerCase();
      const region = (wine.region || '').toLowerCase();

      return name.includes(queryLower) ||
             winery.includes(queryLower) ||
             region.includes(queryLower);
    });

    // Apply filters
    wines = wines.filter(wine => {
      const price = parseFloat(wine.price) || 0;
      const rating = parseFloat(wine.rating) || 0;

      return rating >= minRating && price <= maxPrice;
    });

    // Limit results
    wines = wines.slice(0, limit);

    // Transform to standard wine format
    const formattedWines = wines.map(wine => ({
      name: wine.wine || 'Unknown',
      vintage: wine.year || '',
      price: parseFloat(wine.price) || null,
      rating: parseFloat(wine.rating) || null,
      country: wine.country || '',
      region: wine.region || '',
      grapes: wine.grape_composition || '',
      alcohol: wine.alcohol_percentage || '',
      url: `https://www.vivino.com/search/wines?q=${encodeURIComponent(wine.wine)}`,
      availability: 'Historical Data (May 2023)',
      source: 'WineSensed (UC Davis/NeurIPS)',
      vintageId: wine.vintage_id || ''
    }));

    return {
      success: true,
      wines: formattedWines,
      count: formattedWines.length,
      source: 'WineSensed (UC Davis/NeurIPS)',
      timestamp: new Date().toISOString()
    };
  }

  /**
   * Get dataset statistics
   */
  getStats() {
    return {
      datasetInfo: this.datasetInfo,
      projectUrl: this.projectUrl,
      downloadUrl: this.datasetUrl,
      citation: 'Bender et al. (2023). Learning to Taste: A Multimodal Wine Dataset. NeurIPS 2023. arXiv:2308.16900',
      features: [
        '350,000+ unique wine vintages',
        '824,000 wine reviews from Vivino',
        '897,000 wine label images',
        'Historical pricing data (May 2023)',
        'Grape composition and varietal info',
        'Regional and country data',
        'Rating and quality scores',
        'Alcohol percentage',
        'Vintage years 1950-2021'
      ],
      useCases: [
        'Historical price trend analysis',
        'Vintage quality comparison',
        'Regional wine characteristics',
        'Price-to-quality correlation',
        'Wine recommendation systems',
        'Academic research'
      ]
    };
  }

  /**
   * Check if cache is valid
   */
  isCacheValid() {
    if (!this.dataCache || !this.cacheTimestamp) {
      return false;
    }
    return (Date.now() - this.cacheTimestamp) < this.cacheDuration;
  }
}

module.exports = new WineSensedScraper();