`routes/scraper.js`

425 lines
const express = require('express');
const router = express.Router();
const klWineScraper = require('../services/klWineScraper');
const vivinoScraper = require('../services/vivinoScraper');
const totalWineScraper = require('../services/totalWineScraper');
const wineAggregator = require('../services/wineAggregator');
const historicalWineData = require('../services/historicalWineData');
const googleSheets = require('../services/googleSheets');
const slackNotifier = require('../services/slackNotifier');

// Price spike threshold (percentage)
const PRICE_SPIKE_THRESHOLD = parseFloat(process.env.PRICE_SPIKE_THRESHOLD) || 20;

// Generate demo wine data
function generateDemoWines(query) {
  const wineries = ['Château Margaux', 'Opus One', 'Caymus', 'Silver Oak', 'Stags Leap', 'Jordan',
                    'Duckhorn', 'Shafer', 'Heitz Cellar', 'Ridge', 'Beringer', 'Robert Mondavi'];
  const vintages = ['2018', '2019', '2020', '2021', '2022'];
  const types = ['Cabernet Sauvignon', 'Pinot Noir', 'Chardonnay', 'Merlot', 'Zinfandel'];
  const regions = ['Napa Valley', 'Sonoma', 'Paso Robles', 'Santa Barbara', 'Willamette Valley'];

  const wines = [];
  const searchLower = query.toLowerCase();

  for (let i = 0; i < 12; i++) {
    const winery = wineries[i % wineries.length];
    const vintage = vintages[Math.floor(Math.random() * vintages.length)];
    const type = types[Math.floor(Math.random() * types.length)];
    const region = regions[Math.floor(Math.random() * regions.length)];
    const price = (Math.random() * 150 + 20).toFixed(2);
    const rating = (Math.random() * 5 + 92).toFixed(0);

    wines.push({
      name: `${winery} ${vintage} ${type}`,
      price: parseFloat(price),
      vintage: vintage,
      rating: parseInt(rating),
      source: 'K&L Wine Merchants',
      availability: Math.random() > 0.3 ? 'In Stock' : 'Limited',
      url: `https://www.klwines.com/product/${i + 1000}`,
      region: region,
      type: type,
      winery: winery
    });
  }

  return wines;
}

// Search for wines (GET - for verify page)
router.get('/search', async (req, res) => {
  try {
    let { query, winery, sources, sortBy } = req.query;

    // Convert sources from comma-separated string to array
    if (sources && typeof sources === 'string') {
      sources = sources.split(',').map(s => s.trim()).filter(s => s);
    }

    // Sanitize input
    if (winery && typeof winery === 'string') {
      query = winery.trim();
    } else if (query && typeof query === 'string') {
      query = query.trim();
    } else {
      return res.status(400).json({
        success: false,
        error: 'Invalid search query'
      });
    }

    if (!query) {
      return res.status(400).json({
        success: false,
        error: 'Search query cannot be empty'
      });
    }

    // Parse sources (default to all)
    if (!sources || !Array.isArray(sources) || sources.length === 0) {
      sources = ['kl', 'vivino', 'totalwine'];
    }

    // Perform search across all specified sources
    let result = await wineAggregator.searchAll(query, {
      sources,
      deduplicate: true,
      sortBy: sortBy || 'price',
      limit: 100
    });

    // If all scrapers fail, return demo data
    if (!result.success || result.wines.length === 0) {
      console.log('All scrapers failed or no results, returning demo data');
      result = {
        success: true,
        wines: generateDemoWines(query),
        count: 12,
        source: 'Demo Data',
        sources: [{ source: 'demo', count: 12 }],
        timestamp: new Date().toISOString()
      };
    }

    // Log to Google Sheets and check for price spikes
    const alerts = [];

    if (result.wines.length > 0) {
      // Log all wines to Google Sheets
      await googleSheets.logMultipleWines(result.wines);

      // Check each wine for price spikes
      for (const wine of result.wines) {
        try {
          const lastRecord = await googleSheets.getLastPrice(wine.name);

          if (lastRecord && lastRecord.price) {
            const oldPrice = lastRecord.price;
            const newPrice = wine.price;
            const percentageChange = ((newPrice - oldPrice) / oldPrice) * 100;

            // Check if price spiked above threshold
            if (percentageChange >= PRICE_SPIKE_THRESHOLD) {
              const alert = {
                wine: wine.name,
                oldPrice,
                newPrice,
                percentageIncrease: percentageChange
              };

              alerts.push(alert);

              // Send Slack notification
              await slackNotifier.sendPriceSpikeAlert(
                wine,
                oldPrice,
                newPrice,
                percentageChange
              );
            }
          }
        } catch (err) {
          console.error('Error checking price spike for wine:', err.message);
        }
      }
    }

    res.json({
      ...result,
      alerts,
      logged: result.wines.length > 0
    });

  } catch (error) {
    console.error('Search error:', error);
    res.status(500).json({
      success: false,
      error: 'Search failed: ' + error.message
    });
  }
});

// Search for wines (POST - for main app)
router.post('/search', async (req, res) => {
  try {
    let { query, winery, sources, sortBy } = req.body;

    // Sanitize input
    if (winery && typeof winery === 'string') {
      query = winery.trim();
    } else if (query && typeof query === 'string') {
      query = query.trim();
    } else {
      return res.status(400).json({
        success: false,
        error: 'Invalid search query'
      });
    }

    if (!query) {
      return res.status(400).json({
        success: false,
        error: 'Search query cannot be empty'
      });
    }

    // Parse sources (default to all)
    if (!sources || !Array.isArray(sources) || sources.length === 0) {
      sources = ['kl', 'vivino', 'totalwine'];
    }

    // Perform search across all specified sources
    let result = await wineAggregator.searchAll(query, {
      sources,
      deduplicate: true,
      sortBy: sortBy || 'price',
      limit: 100
    });

    // If all scrapers fail, return demo data
    if (!result.success || result.wines.length === 0) {
      console.log('All scrapers failed or no results, returning demo data');
      result = {
        success: true,
        wines: generateDemoWines(query),
        count: 12,
        source: 'Demo Data',
        sources: [{ source: 'demo', count: 12 }],
        timestamp: new Date().toISOString()
      };
    }

    // Log to Google Sheets and check for price spikes
    const alerts = [];

    if (result.wines.length > 0) {
      // Log all wines to Google Sheets
      await googleSheets.logMultipleWines(result.wines);

      // Check each wine for price spikes
      for (const wine of result.wines) {
        try {
          const lastRecord = await googleSheets.getLastPrice(wine.name);

          if (lastRecord && lastRecord.price) {
            const oldPrice = lastRecord.price;
            const newPrice = wine.price;
            const percentageChange = ((newPrice - oldPrice) / oldPrice) * 100;

            // Check if price spiked above threshold
            if (percentageChange >= PRICE_SPIKE_THRESHOLD) {
              const alert = {
                wine: wine.name,
                oldPrice,
                newPrice,
                percentageIncrease: percentageChange
              };

              alerts.push(alert);

              // Send Slack notification
              await slackNotifier.sendPriceSpikeAlert(
                wine,
                oldPrice,
                newPrice,
                percentageChange
              );
            }
          }
        } catch (err) {
          console.error('Error checking price spike for wine:', err.message);
        }
      }
    }

    res.json({
      ...result,
      alerts,
      logged: result.wines.length > 0
    });

  } catch (error) {
    console.error('Search error:', error);
    res.status(500).json({
      success: false,
      error: 'Search failed: ' + error.message
    });
  }
});

// Quick search for FOXEN
router.get('/foxen', async (req, res) => {
  try {
    const result = await klWineScraper.searchFoxen();

    if (result.wines.length > 0) {
      await googleSheets.logMultipleWines(result.wines);
    }

    res.json({
      ...result,
      logged: result.wines.length > 0
    });

  } catch (error) {
    console.error('FOXEN search error:', error);
    res.status(500).json({
      success: false,
      error: 'Search failed: ' + error.message
    });
  }
});

// Quick search for NAVARRO
router.get('/navarro', async (req, res) => {
  try {
    const result = await klWineScraper.searchNavarro();

    if (result.wines.length > 0) {
      await googleSheets.logMultipleWines(result.wines);
    }

    res.json({
      ...result,
      logged: result.wines.length > 0
    });

  } catch (error) {
    console.error('NAVARRO search error:', error);
    res.status(500).json({
      success: false,
      error: 'Search failed: ' + error.message
    });
  }
});

// Get best prices across all sources
router.post('/best-price', async (req, res) => {
  try {
    const { query } = req.body;

    if (!query || typeof query !== 'string') {
      return res.status(400).json({
        success: false,
        error: 'Invalid search query'
      });
    }

    const result = await wineAggregator.findBestPrice(query.trim());
    res.json(result);

  } catch (error) {
    console.error('Best price search error:', error);
    res.status(500).json({
      success: false,
      error: 'Search failed: ' + error.message
    });
  }
});

// Get available wine sources
router.get('/sources', (req, res) => {
  const sources = wineAggregator.getAvailableSources();
  res.json({
    success: true,
    sources: sources.map(s => ({
      id: s.id,
      name: s.name
    }))
  });
});

// Search specific source
router.post('/source/:sourceId', async (req, res) => {
  try {
    const { sourceId } = req.params;
    const { query } = req.body;

    if (!query || typeof query !== 'string') {
      return res.status(400).json({
        success: false,
        error: 'Invalid search query'
      });
    }

    const result = await wineAggregator.searchSource(sourceId, query.trim());

    if (result.success && result.wines.length > 0) {
      await googleSheets.logMultipleWines(result.wines);
    }

    res.json(result);

  } catch (error) {
    console.error(`Source ${req.params.sourceId} search error:`, error);
    res.status(500).json({
      success: false,
      error: 'Search failed: ' + error.message
    });
  }
});

// Get historical wine dataset information
router.get('/dataset-info', (req, res) => {
  try {
    const info = historicalWineData.getDatasetInfo();
    res.json({
      success: true,
      dataset: info
    });
  } catch (error) {
    console.error('Dataset info error:', error);
    res.status(500).json({
      success: false,
      error: error.message
    });
  }
});

// Search historical wine data
router.post('/historical', async (req, res) => {
  try {
    const { query, options } = req.body;

    if (!query || typeof query !== 'string') {
      return res.status(400).json({
        success: false,
        error: 'Invalid search query'
      });
    }

    const result = await historicalWineData.searchHistorical(query.trim(), options);
    res.json(result);

  } catch (error) {
    console.error('Historical search error:', error);
    res.status(500).json({
      success: false,
      error: 'Historical search failed: ' + error.message
    });
  }
});

module.exports = router;