← back to Watches

arte-specs-scraper.js

255 lines

const puppeteer = require('/root/Projects/watches/node_modules/puppeteer');
const { Pool } = require('/root/Projects/watches/node_modules/pg');
const https = require('https');

// PostgreSQL connection
const pool = new Pool({
  connectionString: (process.env.DATABASE_URL || 'postgresql://dw_admin@127.0.0.1:5432/dw_unified')
});

// Slack webhook for completion notification
const SLACK_WEBHOOK = 'https://hooks.slack.com/services/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';

function sendSlackMessage(message) {
  return new Promise((resolve, reject) => {
    const payload = JSON.stringify({ text: message });
    const options = {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Content-Length': payload.length
      }
    };

    const req = https.request(SLACK_WEBHOOK, options, (res) => {
      resolve();
    });

    req.on('error', (e) => {
      console.error('Slack notification failed:', e.message);
      resolve(); // Don't fail the script if Slack fails
    });

    req.write(payload);
    req.end();
  });
}

async function scrapeArteSpecs() {
  const startTime = Date.now();
  console.log('🚀 Starting Arte specs scraper...\n');

  // Launch browser
  const browser = await puppeteer.launch({
    executablePath: '/root/.cache/puppeteer/chrome/linux-145.0.7632.46/chrome-linux64/chrome',
    headless: 'new',
    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage']
  });

  const page = await browser.newPage();
  await page.setViewport({ width: 1920, height: 1080 });

  // Query all Arte products
  const result = await pool.query(`
    SELECT arte_sku, product_url, description, specs
    FROM arte_catalog
    WHERE product_url IS NOT NULL AND product_url != ''
    ORDER BY arte_sku
  `);

  const products = result.rows;
  console.log(`📦 Found ${products.length} Arte products to scrape\n`);

  let successCount = 0;
  let errorCount = 0;
  const errors = [];

  for (let i = 0; i < products.length; i++) {
    const product = products[i];
    const { arte_sku, product_url } = product;

    try {
      console.log(`[${i + 1}/${products.length}] Scraping ${arte_sku}...`);

      // Navigate to product page
      await page.goto(product_url, { waitUntil: 'networkidle2', timeout: 30000 });

      // Extract description
      let description = null;
      try {
        description = await page.$eval('div.wysiwyg.mb-4.font-light.max-w-xl', el => el.textContent.trim());
      } catch (e) {
        console.log(`  ⚠️  No description found`);
      }

      // Extract all specs - try multiple strategies
      const specs = await page.evaluate(() => {
        const specsObj = {};

        // Strategy 1: Look for ALL dl elements on the page
        const allDls = document.querySelectorAll('dl');
        allDls.forEach(dl => {
          const dts = dl.querySelectorAll('dt');
          const dds = dl.querySelectorAll('dd');

          // Match dt with corresponding dd
          dts.forEach((dt, idx) => {
            if (dds[idx]) {
              const key = dt.textContent.trim();
              const value = dds[idx].textContent.trim();
              if (key && value && key.length < 100) { // Reasonable key length
                specsObj[key] = value;
              }
            }
          });
        });

        // Strategy 2: If no specs found, look for grid/flex layouts with label-value pairs
        if (Object.keys(specsObj).length === 0) {
          // Look for divs with grid or flex that might contain specs
          const containers = document.querySelectorAll('div[class*="grid"], div[class*="flex"], section');

          containers.forEach(container => {
            const textNodes = Array.from(container.children || []);

            // Look for colon-separated patterns like "Width: 70 cm"
            textNodes.forEach(node => {
              const text = node.textContent.trim();
              if (text.includes(':')) {
                const [key, ...valueParts] = text.split(':');
                const value = valueParts.join(':').trim();
                if (key && value && key.length < 50 && value.length < 200) {
                  specsObj[key.trim()] = value;
                }
              }
            });
          });
        }

        // Strategy 3: Look for specific known spec patterns in all text
        if (Object.keys(specsObj).length === 0) {
          const bodyText = document.body.textContent;

          // Common patterns to extract
          const patterns = [
            /Width[:\s]+([0-9.]+\s*(?:cm|m|inches)?)/i,
            /Length[:\s]+([0-9.]+\s*(?:cm|m|meters)?)/i,
            /Weight[:\s]+([0-9.]+\s*(?:g|kg|gr\/m²)?)/i,
            /Composition[:\s]+([^\.]+)/i,
            /Material[:\s]+([^\.]+)/i,
            /Pattern Repeat[:\s]+([0-9.]+\s*(?:cm|m)?)/i,
            /Fire Rating[:\s]+([A-Z0-9\s-]+)/i
          ];

          patterns.forEach(pattern => {
            const match = bodyText.match(pattern);
            if (match) {
              const key = match[0].split(/[:\s]/)[0];
              specsObj[key] = match[1].trim();
            }
          });
        }

        return specsObj;
      });

      // Extract individual spec fields for dedicated columns
      const width = specs['Width'] || specs['Largeur'] || null;
      const composition = specs['Composition'] || specs['Material'] || specs['Matériau'] || null;
      const patternRepeat = specs['Pattern Repeat'] || specs['Rapport'] || specs['Repeat'] || null;
      const weight = specs['Weight'] || specs['Poids'] || null;
      const fireRatingUS = specs['Fire Rating'] || specs['Fire Classification (US)'] || null;
      const fireRatingEU = specs['Fire Rating (EU)'] || specs['Fire Classification (EU)'] || null;

      console.log(`  ✅ Found ${Object.keys(specs).length} specs`);
      if (description) {
        console.log(`  ✅ Found description (${description.length} chars)`);
      }

      // Update PostgreSQL
      await pool.query(`
        UPDATE arte_catalog
        SET description = COALESCE($1, description),
            specs = $2,
            width = COALESCE($3, width),
            composition = COALESCE($4, composition),
            pattern_repeat = COALESCE($5, pattern_repeat),
            weight = COALESCE($6, weight),
            fire_rating_us = COALESCE($7, fire_rating_us),
            fire_rating_eu = COALESCE($8, fire_rating_eu),
            updated_at = NOW()
        WHERE arte_sku = $9
      `, [
        description,
        JSON.stringify(specs),
        width,
        composition,
        patternRepeat,
        weight,
        fireRatingUS,
        fireRatingEU,
        arte_sku
      ]);

      successCount++;

      // Log progress every 25 products
      if ((i + 1) % 25 === 0) {
        const elapsed = ((Date.now() - startTime) / 1000 / 60).toFixed(1);
        const rate = ((i + 1) / elapsed).toFixed(1);
        console.log(`\n📊 Progress: ${i + 1}/${products.length} (${((i + 1) / products.length * 100).toFixed(1)}%) | ${elapsed}min | ${rate} products/min | Success: ${successCount}, Errors: ${errorCount}\n`);
      }

      // Rate limit: 1.5 second delay
      await new Promise(resolve => setTimeout(resolve, 1500));

    } catch (error) {
      errorCount++;
      const errorMsg = `${arte_sku}: ${error.message}`;
      errors.push(errorMsg);
      console.log(`  ❌ Error: ${error.message}`);

      // Continue with next product
    }
  }

  await browser.close();
  await pool.end();

  // Final summary
  const elapsed = ((Date.now() - startTime) / 1000 / 60).toFixed(1);
  console.log('\n' + '='.repeat(60));
  console.log('✅ Arte Specs Scraper Complete!');
  console.log('='.repeat(60));
  console.log(`Total Products: ${products.length}`);
  console.log(`Successful: ${successCount}`);
  console.log(`Errors: ${errorCount}`);
  console.log(`Time Elapsed: ${elapsed} minutes`);
  console.log(`Average Rate: ${(successCount / elapsed).toFixed(1)} products/min`);

  if (errors.length > 0) {
    console.log(`\n❌ Errors (${errors.length}):`);
    errors.slice(0, 10).forEach(err => console.log(`  - ${err}`));
    if (errors.length > 10) {
      console.log(`  ... and ${errors.length - 10} more`);
    }
  }

  // Send Slack notification
  const slackMessage = `🎨 *Arte Specs Scraper Complete*\n\n` +
    `• Total Products: ${products.length}\n` +
    `• Successful: ${successCount}\n` +
    `• Errors: ${errorCount}\n` +
    `• Time: ${elapsed} minutes\n` +
    `• Rate: ${(successCount / elapsed).toFixed(1)} products/min`;

  await sendSlackMessage(slackMessage);
  console.log('\n📨 Slack notification sent!');
}

// Run the scraper
scrapeArteSpecs().catch(error => {
  console.error('💥 Fatal error:', error);
  process.exit(1);
});