`services/xWinesScraper.js`

319 lines
const axios = require('axios');
const csv = require('csv-parser');
const { Readable } = require('stream');

/**
 * X-Wines Dataset Scraper
 *
 * A world wines dataset with user ratings for recommendation systems
 * Source: Rogério Xavier de Azambuja / Kaggle / GitHub
 *
 * Dataset Details:
 * - 100,646 wine instances with 17 attributes
 * - 21,013,536 user ratings (5-star scale, 2012-2021)
 * - 1,056,079 anonymized users
 * - Wines from 62 different countries
 *
 * Files:
 * - XWines_100K_wines.csv - Wine attributes and metadata
 * - XWines_21M_ratings.csv - User ratings with timestamps
 *
 * Kaggle: https://www.kaggle.com/datasets/rogerioxavier/x-wines-slim-version
 * GitHub: https://github.com/rogerioxavier/X-Wines
 * License: Open for educational and research use
 */

class XWinesScraper {
  constructor() {
    this.kaggleDataset = 'rogerioxavier/x-wines-slim-version';
    this.githubUrl = 'https://github.com/rogerioxavier/X-Wines';
    this.rawDataUrl = 'https://raw.githubusercontent.com/rogerioxavier/X-Wines/main';

    // Cache for dataset
    this.winesCache = null;
    this.ratingsCache = null;
    this.cacheTimestamp = null;
    this.cacheDuration = 24 * 60 * 60 * 1000; // 24 hours

    this.datasetInfo = {
      name: 'X-Wines Dataset',
      source: 'Rogério Xavier (Open Web 2022)',
      wines: 100646,
      ratings: 21013536,
      users: 1056079,
      countries: 62,
      ratingPeriod: '2012-2021',
      license: 'Open for educational/research use'
    };
  }

  /**
   * Search X-Wines dataset for wines
   * Note: This requires the CSV dataset to be downloaded locally or accessed via GitHub
   */
  async search(query, options = {}) {
    try {
      const {
        limit = 50,
        minRating = 0,
        maxPrice = 10000,
        country = null,
        minYear = 1900,
        maxYear = 2023
      } = options;

      // For now, return dataset information with instructions
      // In production, this would query the downloaded CSV data or GitHub raw files

      return {
        success: true,
        wines: [],
        count: 0,
        source: 'X-Wines Dataset',
        message: 'X-Wines dataset requires local download or Kaggle API access. Dataset contains 100k wines with 21M ratings.',
        datasetInfo: this.datasetInfo,
        downloadInstructions: {
          option1: {
            method: 'Kaggle Download',
            url: 'https://www.kaggle.com/datasets/rogerioxavier/x-wines-slim-version',
            steps: [
              'Create Kaggle account',
              'Download XWines_100K_wines.csv',
              'Download XWines_21M_ratings.csv',
              'Place files in /data/xwines/ directory'
            ]
          },
          option2: {
            method: 'Kaggle API',
            command: 'kaggle datasets download -d rogerioxavier/x-wines-slim-version',
            requiresSetup: 'Install kaggle CLI and configure API token'
          },
          option3: {
            method: 'GitHub Clone',
            url: 'https://github.com/rogerioxavier/X-Wines',
            note: 'May contain sample data only'
          }
        },
        timestamp: new Date().toISOString()
      };

    } catch (error) {
      console.error('X-Wines scraper error:', error.message);
      return {
        success: false,
        wines: [],
        error: error.message,
        source: 'X-Wines'
      };
    }
  }

  /**
   * Load wines CSV from local file
   */
  async loadWinesDataset(csvPath) {
    return new Promise((resolve, reject) => {
      const results = [];
      const fs = require('fs');

      if (!fs.existsSync(csvPath)) {
        return reject(new Error(`Dataset file not found: ${csvPath}`));
      }

      fs.createReadStream(csvPath)
        .pipe(csv())
        .on('data', (data) => results.push(data))
        .on('end', () => {
          this.winesCache = results;
          this.cacheTimestamp = Date.now();
          resolve(results);
        })
        .on('error', reject);
    });
  }

  /**
   * Load ratings CSV from local file
   */
  async loadRatingsDataset(csvPath) {
    return new Promise((resolve, reject) => {
      const results = [];
      const fs = require('fs');

      if (!fs.existsSync(csvPath)) {
        return reject(new Error(`Dataset file not found: ${csvPath}`));
      }

      fs.createReadStream(csvPath)
        .pipe(csv())
        .on('data', (data) => results.push(data))
        .on('end', () => {
          this.ratingsCache = results;
          resolve(results);
        })
        .on('error', reject);
    });
  }

  /**
   * Search loaded wines dataset
   */
  async searchLoadedData(query, options = {}) {
    if (!this.winesCache) {
      return {
        success: false,
        wines: [],
        error: 'Dataset not loaded. Call loadWinesDataset() first.',
        source: 'X-Wines'
      };
    }

    const {
      limit = 50,
      minRating = 0,
      maxPrice = 10000,
      country = null
    } = options;

    const queryLower = query.toLowerCase();

    // Filter wines by query
    let wines = this.winesCache.filter(wine => {
      const name = (wine.WineName || wine.name || '').toLowerCase();
      const winery = (wine.Winery || wine.winery || '').toLowerCase();
      const region = (wine.Region || wine.region || '').toLowerCase();
      const wineCountry = (wine.Country || wine.country || '').toLowerCase();

      const matchesQuery = name.includes(queryLower) ||
                          winery.includes(queryLower) ||
                          region.includes(queryLower);

      const matchesCountry = !country || wineCountry === country.toLowerCase();

      return matchesQuery && matchesCountry;
    });

    // Apply rating and price filters if available
    wines = wines.filter(wine => {
      const rating = parseFloat(wine.Rating || wine.rating || 0);
      const price = parseFloat(wine.Price || wine.price || 0);

      return rating >= minRating && price <= maxPrice;
    });

    // Limit results
    wines = wines.slice(0, limit);

    // Transform to standard wine format
    const formattedWines = wines.map(wine => ({
      name: wine.WineName || wine.name || 'Unknown',
      vintage: wine.Year || wine.year || '',
      price: parseFloat(wine.Price || wine.price) || null,
      rating: parseFloat(wine.Rating || wine.rating) || null,
      country: wine.Country || wine.country || '',
      region: wine.Region || wine.region || '',
      winery: wine.Winery || wine.winery || '',
      type: wine.Type || wine.type || '',
      grapes: wine.Grapes || wine.grapes || '',
      url: wine.URL || wine.url || '',
      availability: 'User-rated wines (2012-2021)',
      source: 'X-Wines Dataset',
      wineId: wine.WineID || wine.wine_id || wine.id || ''
    }));

    return {
      success: true,
      wines: formattedWines,
      count: formattedWines.length,
      source: 'X-Wines Dataset',
      timestamp: new Date().toISOString()
    };
  }

  /**
   * Get wine ratings for a specific wine
   */
  async getWineRatings(wineId) {
    if (!this.ratingsCache) {
      return {
        success: false,
        ratings: [],
        error: 'Ratings dataset not loaded. Call loadRatingsDataset() first.'
      };
    }

    const ratings = this.ratingsCache.filter(r =>
      (r.WineID || r.wine_id || r.id) === wineId
    );

    return {
      success: true,
      wineId,
      ratings: ratings.map(r => ({
        userId: r.UserID || r.user_id,
        rating: parseFloat(r.Rating || r.rating),
        date: r.Date || r.date || r.timestamp
      })),
      count: ratings.length,
      averageRating: ratings.length > 0
        ? ratings.reduce((sum, r) => sum + parseFloat(r.Rating || r.rating || 0), 0) / ratings.length
        : null
    };
  }

  /**
   * Get dataset statistics
   */
  getStats() {
    return {
      datasetInfo: this.datasetInfo,
      githubUrl: this.githubUrl,
      kaggleUrl: `https://www.kaggle.com/datasets/${this.kaggleDataset}`,
      citation: 'Xavier, R. X-Wines: A Wine Dataset for Recommender Systems and Machine Learning. Big Data and Cognitive Computing, 2023.',
      features: [
        '100,646 wine instances',
        '21 million user ratings',
        '1 million+ anonymized users',
        'Wines from 62 countries',
        'Rating period: 2012-2021',
        '17 wine attributes per instance',
        'Temporal rating data',
        'User-generated ratings'
      ],
      attributes: [
        'Wine name',
        'Winery',
        'Year/Vintage',
        'Price',
        'Rating',
        'Country',
        'Region',
        'Type (red, white, etc.)',
        'Grapes/Varietals'
      ],
      useCases: [
        'Wine recommendation systems',
        'Collaborative filtering',
        'Rating prediction',
        'User preference analysis',
        'Wine popularity trends',
        'Price-rating correlation',
        'Geographic wine analysis',
        'Machine learning research'
      ]
    };
  }

  /**
   * Check if cache is valid
   */
  isCacheValid() {
    if (!this.winesCache || !this.cacheTimestamp) {
      return false;
    }
    return (Date.now() - this.cacheTimestamp) < this.cacheDuration;
  }
}

module.exports = new XWinesScraper();