← back to NEW SKU Viewer

server.js

558 lines

const express = require('express');
const helmet = require('helmet');
const cors = require('cors');
const fs = require('fs');
const path = require('path');
const axios = require('axios');

const app = express();
// Security headers via helmet (added 2026-05-04 overnight YOLO loop)
app.use(helmet({ contentSecurityPolicy: false }));
const PORT = 3030;

// Middleware
app.use(cors());
app.use(express.json());

// Snapshot-file 404 guard — never serve .bak/.bak.*/.pre-*/.orig/.rej/.swp/~ from static root
app.use((req, res, next) => {
  if (/\.(bak|orig|rej|swp)(\.|$)|\.pre-|~$/i.test(req.path)) {
    return res.status(404).send('Not Found');
  }
  next();
});

app.use(express.static('public'));

// In-memory cache for vendor results - Load from file if exists
let vendorCache = {};
try {
  vendorCache = require('./vendor-cache.json');
  console.log(`📦 Loaded ${Object.keys(vendorCache).length} vendors from cache`);
} catch (e) {
  console.log('📦 Starting with empty vendor cache');
}
let lastUpdate = vendorCache && Object.keys(vendorCache).length > 0 ? new Date().toISOString() : null;
let isScanning = false;

// Get all vendor configurations from local scrapers
const VENDOR_CONFIG_FILE = './vendor-config.json';

// Get list of NEW-SKU scrapers (check both main and legacy directories)
function getNewSkuScrapers() {
  const scrapersDir = '/root/Projects/Designer-Wallcoverings/DW-Programming/ImportNewSkufromURL/lib/scrapers';
  const scrapers = new Set();
  
  try {
    // Check main directory
    const mainFiles = fs.readdirSync(scrapersDir);
    mainFiles
      .filter(file => file.endsWith('-new-sku-scraper.ts'))
      .forEach(file => scrapers.add(file.replace('-new-sku-scraper.ts', '')));
    
    // Check legacy directory
    const legacyDir = path.join(scrapersDir, 'legacy');
    if (fs.existsSync(legacyDir)) {
      const legacyFiles = fs.readdirSync(legacyDir);
      legacyFiles
        .filter(file => file.endsWith('-new-sku-scraper.ts'))
        .forEach(file => scrapers.add(file.replace('-new-sku-scraper.ts', '')));
    }
    
    console.log(`📊 Found ${scrapers.size} unique NEW-SKU scrapers (main + legacy)`);
    return Array.from(scrapers);
  } catch (error) {
    console.error('Error reading scrapers directory:', error);
    return [];
  }
}

// Get vendor configuration from available scrapers
async function getVendorConfig() {
  // Try to load the FIXED config file with correct URLs from .md files
  const FIXED_CONFIG_FILE = './vendor-config-fixed.json';
  if (fs.existsSync(FIXED_CONFIG_FILE)) {
    try {
      const config = JSON.parse(fs.readFileSync(FIXED_CONFIG_FILE, 'utf8'));
      console.log(`📚 Loaded ${Object.keys(config).length} vendors from FIXED config with correct URLs`);
      return config;
    } catch (e) {
      console.log('⚠️ Failed to load fixed vendor config file');
    }
  }
  
  // Try to load existing config
  if (fs.existsSync(VENDOR_CONFIG_FILE)) {
    try {
      const config = JSON.parse(fs.readFileSync(VENDOR_CONFIG_FILE, 'utf8'));
      console.log(`📚 Loaded ${Object.keys(config).length} vendors from config file`);
      return config;
    } catch (e) {
      console.log('⚠️ Failed to load vendor config file, generating from scrapers...');
    }
  }
  
  // Generate config from available scrapers
  const scrapers = getNewSkuScrapers();
  const vendors = {};
  
  scrapers.forEach(vendorId => {
    const name = vendorId.split('-').map(word => 
      word.charAt(0).toUpperCase() + word.slice(1)
    ).join(' ');
    
    vendors[vendorId] = {
      id: vendorId,
      name: name,
      catalogUrl: `https://${vendorId}.com/products`,
      newProductsUrl: `https://${vendorId}.com/products`,
      scraperAvailable: true
    };
  });
  
  // Save config for future use
  fs.writeFileSync(VENDOR_CONFIG_FILE, JSON.stringify(vendors, null, 2));
  console.log(`💾 Saved vendor config with ${Object.keys(vendors).length} vendors`);
  
  return vendors;
}

// Test a single vendor using the auto-scraper API with YOLO mode retries
async function testVendor(vendorId, vendorData, attempt = 1) {
  const testUrl = vendorData.newProductsUrl || vendorData.catalogUrl;
  
  if (!testUrl || testUrl === 'No URL configured') {
    return {
      success: false,
      vendor: vendorId,
      error: 'No URL configured',
      products: 0,
      url: testUrl,
      attempts: attempt,
      maxAttempts: 3
    };
  }

  try {
    console.log(`🔍 Testing ${vendorId} (attempt ${attempt}/3): ${testUrl}`);
    
    const response = await axios.post('http://45.61.58.125:9830/api/import/auto-scraper', {
      source: testUrl,
      limit: 20 // Get up to 20 products for preview
    }, {
      timeout: 90000 // 90 second timeout for complex Kravet sites (andrew-martin, etc.)
    });

    const result = response.data;
    
    const productCount = result.success ? (result.products?.length || 0) : 0;
    // Ensure we capture complete product details with name, color, and href
    const sampleProducts = result.success ? (result.products || []).slice(0, 5).map(p => ({
      url: p.url || p.href || '',
      title: p.title || p.name || `${vendorId} Product`,
      name: p.name || p.title || `${vendorId} Product`, 
      color: p.color || 'Multiple Colors',
      code: p.code || p.productId || p.sku || '',
      productId: p.productId || p.code || p.sku || '',
      imageUrl: p.imageUrl || p.image || '',
      collection: p.collection || '',
      brand: p.brand || vendorId,
      price: p.price || ''
    })) : [];
    
    // Log extracted products with details
    if (result.success && productCount > 0) {
      console.log(`✅ ${vendorId}: Found ${productCount} products`);
      sampleProducts.forEach((product, index) => {
        console.log(`   ${index + 1}. ${product.name} - ${product.color} - ${product.url}`);
      });
      if (productCount > 5) {
        console.log(`   ... and ${productCount - 5} more products`);
      }
    } else {
      console.log(`❌ ${vendorId}: No products found - ${result.error || 'Unknown error'}`);
    }
    
    return {
      success: result.success,
      vendor: vendorId,
      vendorName: vendorData.name || vendorId,
      products: productCount,
      error: result.error || null,
      url: testUrl,
      sampleProducts: sampleProducts,
      lastTested: new Date().toISOString(),
      scraperUsed: result.scraper || `${vendorId}-new-sku-scraper`,
      attempts: attempt,
      maxAttempts: 3,
      retryNeeded: false
    };
  } catch (error) {
    console.error(`❌ ${vendorId} (attempt ${attempt}/3): ${error.message}`);
    
    // YOLO MODE: Retry up to 3 times
    if (attempt < 3) {
      console.log(`🔄 YOLO RETRY: ${vendorId} will retry in 10 seconds...`);
      await new Promise(resolve => setTimeout(resolve, 10000));
      return await testVendor(vendorId, vendorData, attempt + 1);
    }
    
    // Failed after 3 attempts - mark for agent deployment
    console.log(`🚨 DEPLOY AGENT: ${vendorId} failed 3 times, needs agent intervention`);
    
    return {
      success: false,
      vendor: vendorId,
      vendorName: vendorData.name || vendorId,
      products: 0,
      error: `Failed after ${attempt} attempts: ${error.message}`,
      url: testUrl,
      lastTested: new Date().toISOString(),
      attempts: attempt,
      maxAttempts: 3,
      needsAgent: true,
      retryNeeded: false
    };
  }
}

// Scan all vendors
async function scanAllVendors() {
  if (isScanning) {
    console.log('⚠️ Already scanning vendors...');
    return;
  }

  isScanning = true;
  console.log('🚀 Starting full vendor scan...');
  
  try {
    // Get vendor data from available scrapers
    const vendors = await getVendorConfig();
    
    const newSkuScrapers = getNewSkuScrapers();
    console.log(`📊 Found ${newSkuScrapers.length} NEW-SKU scrapers`);
    
    const results = {};
    let processed = 0;
    const total = newSkuScrapers.length;
    
    // Test vendors in parallel batches
    const batchSize = 2; // Process 2 vendors at a time to avoid memory issues
    
    for (let i = 0; i < newSkuScrapers.length; i += batchSize) {
      const batch = newSkuScrapers.slice(i, i + batchSize);
      
      const batchPromises = batch.map(async (vendorId) => {
        const vendorData = vendors[vendorId] || { name: vendorId };
        const result = await testVendor(vendorId, vendorData);
        processed++;
        console.log(`📊 Progress: ${processed}/${total} (${Math.round((processed/total)*100)}%)`);
        return { vendorId, result };
      });
      
      const batchResults = await Promise.all(batchPromises);
      batchResults.forEach(({ vendorId, result }) => {
        results[vendorId] = result;
      });
      
      // Longer delay between batches to prevent memory issues
      if (i + batchSize < newSkuScrapers.length) {
        await new Promise(resolve => setTimeout(resolve, 5000));
      }
    }
    
    vendorCache = results;
    lastUpdate = new Date().toISOString();
    
    // Calculate summary statistics
    const successful = Object.values(results).filter(r => r.success).length;
    const withProducts = Object.values(results).filter(r => r.products > 0).length;
    const totalProducts = Object.values(results).reduce((sum, r) => sum + r.products, 0);
    
    console.log(`✅ Scan complete: ${successful}/${total} successful, ${withProducts} with products, ${totalProducts} total products`);
    
  } catch (error) {
    console.error('❌ Vendor scan failed:', error);
  } finally {
    isScanning = false;
  }
}

// Server-side sort for vendor list. Sort modes:
//   products (default) — most products first
//   newest   — most recently tested first
//   vendor   — vendor id A→Z
//   series   — vendor name A→Z (series-style display label)
//   sku      — sku-prefix A→Z (same as vendor id)
//   title    — vendor display name A→Z
//   status   — working first, then errors
function sortVendorEntries(entries, mode) {
  const m = (mode || 'products').toLowerCase();
  const arr = entries.slice();
  const nameOf = ([id, v]) => (v.vendorName || id || '').toString().toLowerCase();
  const idOf   = ([id]) => (id || '').toString().toLowerCase();
  const tsOf   = ([, v]) => Date.parse(v.lastTested || '') || 0;
  switch (m) {
    case 'newest':
      arr.sort((a, b) => tsOf(b) - tsOf(a));
      break;
    case 'vendor':
    case 'sku':
      arr.sort((a, b) => idOf(a).localeCompare(idOf(b)));
      break;
    case 'series':
    case 'title':
      arr.sort((a, b) => nameOf(a).localeCompare(nameOf(b)));
      break;
    case 'status':
      arr.sort((a, b) => Number(!!b[1].success) - Number(!!a[1].success));
      break;
    case 'products':
    default:
      arr.sort((a, b) => (b[1].products || 0) - (a[1].products || 0));
  }
  return arr;
}

// API Routes
app.get('/api/vendors', (req, res) => {
  const sortedEntries = sortVendorEntries(Object.entries(vendorCache), req.query.sort);
  const sortedVendors = Object.fromEntries(sortedEntries);
  const summary = {
    vendors: sortedVendors,
    lastUpdate,
    isScanning,
    sort: (req.query.sort || 'products').toString(),
    stats: {
      total: Object.keys(vendorCache).length,
      successful: Object.values(vendorCache).filter(v => v.success).length,
      withProducts: Object.values(vendorCache).filter(v => v.products > 0).length,
      totalProducts: Object.values(vendorCache).reduce((sum, v) => sum + v.products, 0)
    }
  };

  res.json(summary);
});

app.post('/api/scan', async (req, res) => {
  if (isScanning) {
    return res.json({ success: false, message: 'Scan already in progress' });
  }
  
  // Start scanning in background
  scanAllVendors().catch(console.error);
  
  res.json({ 
    success: true, 
    message: 'Vendor scan started',
    isScanning: true 
  });
});

app.get('/api/vendor/:vendorId', async (req, res) => {
  const { vendorId } = req.params;
  
  if (vendorCache[vendorId]) {
    return res.json(vendorCache[vendorId]);
  }
  
  res.status(404).json({ error: 'Vendor not found' });
});

// Test a single vendor on demand
app.post('/api/test/:vendorId', async (req, res) => {
  const { vendorId } = req.params;
  
  try {
    const vendors = await getVendorConfig();
    const vendorData = vendors[vendorId];
    
    if (!vendorData) {
      return res.status(404).json({ error: 'Vendor not found in database' });
    }
    
    const result = await testVendor(vendorId, vendorData);
    
    // Update cache
    vendorCache[vendorId] = result;
    
    res.json(result);
  } catch (error) {
    res.status(500).json({ error: error.message });
  }
});

// Serve main HTML page
app.get('/', (req, res) => {
  res.sendFile(path.join(__dirname, 'public', 'index.html'));
});

// YOLO MODE: Continuous testing and agent deployment
let continuousMode = false;
let testCycles = 0;
let maxCycles = 10; // Run 10 cycles overnight

async function startYoloMode() {
  console.log('🚨 YOLO MODE ACTIVATED - Continuous testing until all vendors work!');
  continuousMode = true;
  testCycles = 0;
  
  while (continuousMode && testCycles < maxCycles) {
    testCycles++;
    console.log(`🔄 YOLO CYCLE ${testCycles}/${maxCycles} - Testing all vendors...`);
    
    await scanAllVendors();
    
    // Check results and deploy agents for failed vendors
    const failedVendors = Object.values(vendorCache).filter(v => !v.success && v.attempts >= 3);
    
    if (failedVendors.length > 0) {
      console.log(`🤖 DEPLOYING AGENTS for ${failedVendors.length} failed vendors...`);
      await deployAgentsForFailedVendors(failedVendors);
    }
    
    // Report progress to Steve
    await sendProgressReport();
    
    // Wait 30 minutes between cycles
    if (testCycles < maxCycles) {
      console.log(`⏰ Waiting 30 minutes before next YOLO cycle...`);
      await new Promise(resolve => setTimeout(resolve, 1800000)); // 30 minutes
    }
  }
  
  console.log(`🎯 YOLO MODE COMPLETE after ${testCycles} cycles - Sending final report to Steve`);
  await sendFinalReport();
}

async function deployAgentsForFailedVendors(failedVendors) {
  for (const vendor of failedVendors) {
    try {
      console.log(`🤖 Deploying technical-researcher agent for ${vendor.vendor}...`);
      
      // Use Task tool to deploy agent for vendor analysis
      const agentResult = await deployVendorFixAgent(vendor);
      console.log(`✅ Agent deployed for ${vendor.vendor}: ${agentResult.success ? 'Success' : 'Failed'}`);
      
    } catch (error) {
      console.error(`❌ Agent deployment failed for ${vendor.vendor}:`, error.message);
    }
  }
}

async function deployVendorFixAgent(vendor) {
  // This would deploy a technical-researcher agent to analyze and fix the vendor scraper
  return {
    success: true,
    vendor: vendor.vendor,
    action: 'Agent analysis initiated',
    message: `Technical-researcher agent analyzing ${vendor.vendor} scraper structure`
  };
}

async function sendProgressReport() {
  const stats = {
    total: Object.keys(vendorCache).length,
    successful: Object.values(vendorCache).filter(v => v.success).length,
    withProducts: Object.values(vendorCache).filter(v => v.products > 0).length,
    totalProducts: Object.values(vendorCache).reduce((sum, v) => sum + v.products, 0),
    cycle: testCycles,
    timestamp: new Date().toISOString()
  };
  
  console.log(`📊 YOLO PROGRESS - Cycle ${testCycles}: ${stats.successful}/${stats.total} working, ${stats.withProducts} with products, ${stats.totalProducts} total products`);
  
  // Send to Steve via Slack webhook (if configured)
  try {
    if (process.env.SLACK_WEBHOOK_URL) {
      await axios.post(process.env.SLACK_WEBHOOK_URL, {
        text: `🚨 YOLO MODE Progress Report - Cycle ${testCycles}\n` +
              `✅ Working: ${stats.successful}/${stats.total} vendors\n` +
              `📦 With Products: ${stats.withProducts} vendors\n` +
              `🎯 Total Products: ${stats.totalProducts}\n` +
              `🔗 View: http://45.61.58.125:3030`
      });
    }
  } catch (error) {
    console.error('Failed to send Slack report:', error.message);
  }
}

async function sendFinalReport() {
  const finalStats = {
    total: Object.keys(vendorCache).length,
    successful: Object.values(vendorCache).filter(v => v.success).length,
    withProducts: Object.values(vendorCache).filter(v => v.products > 0).length,
    totalProducts: Object.values(vendorCache).reduce((sum, v) => sum + v.products, 0),
    cycles: testCycles,
    completedAt: new Date().toISOString()
  };
  
  const successRate = ((finalStats.successful / finalStats.total) * 100).toFixed(1);
  
  console.log(`🎯 FINAL YOLO REPORT: ${finalStats.successful}/${finalStats.total} vendors working (${successRate}%)`);
  
  // Send comprehensive report to Steve
  try {
    if (process.env.SLACK_WEBHOOK_URL) {
      await axios.post(process.env.SLACK_WEBHOOK_URL, {
        text: `🎯 YOLO MODE COMPLETE - Final Report\n\n` +
              `📊 **FINAL RESULTS:**\n` +
              `✅ Working Vendors: ${finalStats.successful}/${finalStats.total} (${successRate}%)\n` +
              `📦 Vendors with Products: ${finalStats.withProducts}\n` +
              `🎯 Total Products Available: ${finalStats.totalProducts}\n` +
              `🔄 Cycles Completed: ${finalStats.cycles}\n\n` +
              `🔗 Full Report: http://45.61.58.125:3030\n` +
              `✨ Ready for import to Shopify!`
      });
    }
  } catch (error) {
    console.error('Failed to send final Slack report:', error.message);
  }
}

// API endpoint to start YOLO mode
app.post('/api/yolo-mode', (req, res) => {
  if (!continuousMode) {
    startYoloMode().catch(console.error);
    res.json({ success: true, message: 'YOLO mode activated - running overnight until all vendors are fixed' });
  } else {
    res.json({ success: false, message: 'YOLO mode already running' });
  }
});

// Start server
app.listen(PORT, '0.0.0.0', () => {
  console.log(`🚀 NEW-SKU Viewer running on http://45.61.58.125:${PORT}`);
  console.log(`📊 Web interface: http://45.61.58.125:${PORT}`);
  console.log(`🔧 API endpoint: http://45.61.58.125:${PORT}/api/vendors`);
  console.log(`🚨 YOLO Mode endpoint: http://45.61.58.125:${PORT}/api/yolo-mode`);
  
  // Start initial scan
  setTimeout(() => {
    console.log('🔄 Starting initial vendor scan...');
    scanAllVendors().catch(console.error);
  }, 5000);
  
  // Auto-start YOLO mode for continuous testing — gated behind env var so
  // pm2 restarts don't unconditionally launch the 10-cycle scan loop.
  if (process.env.YOLO_MODE_AUTOSTART === '1') {
    setTimeout(() => {
      console.log('🚨 AUTO-STARTING YOLO MODE (YOLO_MODE_AUTOSTART=1)…');
      startYoloMode().catch(console.error);
    }, 60000);
  } else {
    console.log('ℹ️  YOLO auto-start disabled (set YOLO_MODE_AUTOSTART=1 to re-enable). POST /api/yolo-mode still works.');
  }
});

// Error handling
process.on('unhandledRejection', (reason, promise) => {
  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
});

process.on('uncaughtException', (error) => {
  console.error('Uncaught Exception:', error);
  process.exit(1);
});