← back to Dw Vendor Scrapers Local

scrape-coordonne.js

289 lines

#!/usr/bin/env node
// ============================================================================
// Coordonne Catalog Scraper — Codename: Carmen
// ============================================================================
// Strategy: WordPress REST API only (no individual page scraping needed)
// The WP API provides title, collection, color, image, and product URL
// Total products: ~2,800+ (cat 98 = repeats, cat 206 = murals)
// Note: X-WP-Total headers not returned by Cloudflare, so we paginate until empty
// ============================================================================

const https = require('https');
const { pool, upsertProduct, wait, getCatalogCount, closePool } = require('./scraper-utils');

const TABLE = 'coordonne_catalog';
const BASE_URL = 'https://coordonne.com';
const PER_PAGE = 100;

// Wallpaper categories
const WALLPAPER_CATS = [
  { id: 98, name: 'Repeated Patterns', type: 'wallcovering' },
  { id: 206, name: 'Murals', type: 'mural' },
];

let totalInserted = 0;
let totalSkipped = 0;
let totalErrors = 0;

function fetchJSON(url) {
  return new Promise((resolve, reject) => {
    const req = https.get(url, {
      timeout: 30000,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
        'Accept': 'application/json',
      }
    }, (res) => {
      if (res.statusCode === 301 || res.statusCode === 302) {
        const loc = res.headers.location || '';
        return fetchJSON(loc.startsWith('http') ? loc : `${BASE_URL}${loc}`).then(resolve).catch(reject);
      }
      let data = '';
      res.on('data', chunk => data += chunk);
      res.on('end', () => {
        try {
          const parsed = JSON.parse(data);
          resolve(parsed);
        } catch (e) {
          reject(new Error(`JSON parse error: ${e.message}`));
        }
      });
      res.on('error', reject);
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
  });
}

async function ensureTable() {
  try {
    await pool.query(`SELECT 1 FROM ${TABLE} LIMIT 1`);
    console.log(`  Table "${TABLE}" exists.`);
  } catch (err) {
    console.log(`  Creating table "${TABLE}"...`);
    await pool.query(`
      CREATE TABLE IF NOT EXISTS ${TABLE} (
        id SERIAL PRIMARY KEY, mfr_sku TEXT UNIQUE, dw_sku TEXT,
        pattern_name TEXT, color_name TEXT, collection TEXT, product_type TEXT,
        width TEXT, length TEXT, repeat_v TEXT, repeat_h TEXT, material TEXT,
        price_retail NUMERIC(10,2), price_trade NUMERIC(10,2),
        image_url TEXT, product_url TEXT,
        in_stock BOOLEAN DEFAULT TRUE, discontinued BOOLEAN DEFAULT FALSE,
        shopify_product_id TEXT, last_scraped TIMESTAMPTZ,
        created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW()
      );
    `);
    console.log(`  Table created.`);
  }
}

async function fetchTaxonomyMap(endpoint) {
  const map = {};
  let page = 1;
  while (true) {
    try {
      const items = await fetchJSON(`${BASE_URL}/wp-json/wp/v2/${endpoint}?per_page=100&page=${page}`);
      if (!Array.isArray(items) || items.length === 0) break;
      for (const c of items) {
        map[c.id] = c.name;
      }
      if (items.length < 100) break;
      page++;
      await wait(500, 200);
    } catch (err) {
      // rest_post_invalid_page_number means we're past the last page
      if (err.message && err.message.includes('rest_post_invalid_page_number')) break;
      console.error(`  Error fetching ${endpoint} page ${page}: ${err.message}`);
      break;
    }
  }
  return map;
}

function extractProductData(item, collectionMap, colorMap, productType) {
  const title = item.title ? item.title.rendered : '';
  const link = item.link || '';
  const slug = item.slug || '';

  // Collection from taxonomy
  const collectionIds = item.collection || [];
  let collection = null;
  if (collectionIds.length > 0 && collectionMap[collectionIds[0]]) {
    collection = collectionMap[collectionIds[0]];
  }

  // Color from model_color taxonomy
  const colorIds = item.model_color || [];
  let colorName = null;
  if (colorIds.length > 0 && colorMap[colorIds[0]]) {
    colorName = colorMap[colorIds[0]];
  }
  // Fallback: extract from class_list
  if (!colorName && item.class_list) {
    const classList = Object.values(item.class_list);
    for (const cls of classList) {
      if (typeof cls === 'string' && cls.startsWith('model_color-')) {
        colorName = cls.replace('model_color-', '').replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
      }
    }
  }

  // Image from yoast og_image
  let imageUrl = null;
  if (item.yoast_head_json && item.yoast_head_json.og_image) {
    const ogImages = item.yoast_head_json.og_image;
    if (ogImages.length > 0) imageUrl = ogImages[0].url;
  }

  // SKU: use the WP post ID prefixed with COORD- since the API doesn't expose the Ref code
  // The slug is unique and more descriptive than the post ID
  const mfrSku = `COORD-${item.id}`;

  // In-stock status from class_list
  let inStock = true;
  if (item.class_list) {
    const classList = Object.values(item.class_list);
    if (classList.some(c => typeof c === 'string' && c.includes('outofstock'))) {
      inStock = false;
    }
  }

  return {
    mfr_sku: mfrSku,
    pattern_name: title,
    color_name: colorName,
    collection: collection,
    product_type: productType,
    image_url: imageUrl,
    product_url: link,
    in_stock: inStock,
  };
}

async function main() {
  console.log('='.repeat(70));
  console.log('  COORDONNE WALLPAPER SCRAPER - Carmen');
  console.log('='.repeat(70));
  console.log(`  Source: WordPress REST API (no page scraping needed)`);
  console.log(`  Table: ${TABLE}`);
  console.log('='.repeat(70));

  const startTime = Date.now();
  await ensureTable();
  const beforeCount = await getCatalogCount(TABLE);
  console.log(`  Current products in ${TABLE}: ${beforeCount}\n`);

  // Phase 1: Fetch taxonomy maps
  console.log('[1] Fetching taxonomy maps...');
  const collectionMap = await fetchTaxonomyMap('collection');
  console.log(`  Collections: ${Object.keys(collectionMap).length}`);
  const colorMap = await fetchTaxonomyMap('model_color');
  console.log(`  Colors: ${Object.keys(colorMap).length}`);

  // Phase 2: Fetch all products via API
  console.log('\n[2] Fetching products from WordPress API...');
  let allProducts = [];

  for (const cat of WALLPAPER_CATS) {
    console.log(`\n  Category: ${cat.name} (ID: ${cat.id})`);
    let page = 1;
    let catCount = 0;

    while (page <= 50) { // safety limit
      try {
        const items = await fetchJSON(
          `${BASE_URL}/wp-json/wp/v2/product?product_cat=${cat.id}&per_page=${PER_PAGE}&page=${page}`
        );

        // If we get an error object instead of array, we're past the last page
        if (!Array.isArray(items)) {
          console.log(`  Page ${page}: end of results`);
          break;
        }
        if (items.length === 0) break;

        for (const item of items) {
          allProducts.push(extractProductData(item, collectionMap, colorMap, cat.type));
        }

        catCount += items.length;
        console.log(`  Page ${page}: ${items.length} products (category total: ${catCount})`);

        if (items.length < PER_PAGE) break;
        page++;
        await wait(500, 300); // Be respectful to the server
      } catch (err) {
        if (err.message && err.message.includes('rest_post_invalid_page_number')) {
          console.log(`  Page ${page}: end of results`);
          break;
        }
        console.error(`  Error on page ${page}: ${err.message}`);
        // Try once more after a longer wait
        await wait(3000, 1000);
        page++;
      }
    }

    console.log(`  ${cat.name} total: ${catCount} products`);
  }

  console.log(`\n  Total products fetched: ${allProducts.length}`);

  // Dedup by mfr_sku (some products may be in both categories)
  const seen = new Set();
  const uniqueProducts = [];
  for (const p of allProducts) {
    if (!seen.has(p.mfr_sku)) {
      seen.add(p.mfr_sku);
      uniqueProducts.push(p);
    }
  }
  console.log(`  Unique products after dedup: ${uniqueProducts.length}`);

  // Show samples
  console.log('\n  Sample products:');
  for (const p of uniqueProducts.slice(0, 5)) {
    console.log(`    ${p.mfr_sku.padEnd(15)} | ${(p.pattern_name || '').substring(0, 35).padEnd(35)} | ${(p.collection || 'N/A').substring(0, 20).padEnd(20)} | ${(p.color_name || 'N/A').substring(0, 15)} | img: ${p.image_url ? 'YES' : 'no'}`);
  }

  // Phase 3: Upsert to database
  console.log(`\n[3] Upserting ${uniqueProducts.length} products to ${TABLE}...`);

  for (let i = 0; i < uniqueProducts.length; i++) {
    const p = uniqueProducts[i];
    try {
      const id = await upsertProduct(TABLE, p);
      if (id) totalInserted++;
      else totalSkipped++;
    } catch (err) {
      totalErrors++;
      if (totalErrors <= 5) console.error(`  DB Error for ${p.mfr_sku}: ${err.message}`);
    }
    if ((i + 1) % 500 === 0) console.log(`  Progress: ${i + 1}/${uniqueProducts.length} (inserted: ${totalInserted})`);
  }

  const afterCount = await getCatalogCount(TABLE);
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);

  console.log('\n' + '='.repeat(70));
  console.log('  COORDONNE SCRAPE COMPLETE');
  console.log('='.repeat(70));
  console.log(`  API products:      ${allProducts.length}`);
  console.log(`  Unique (deduped):  ${uniqueProducts.length}`);
  console.log(`  Inserted/updated:  ${totalInserted}`);
  console.log(`  Skipped:           ${totalSkipped}`);
  console.log(`  Errors:            ${totalErrors}`);
  console.log(`  Before:            ${beforeCount}`);
  console.log(`  After:             ${afterCount}`);
  console.log(`  Net new:           ${afterCount - beforeCount}`);
  console.log(`  Duration:          ${elapsed}s`);
  console.log('='.repeat(70));

  await closePool();
}

main().catch(err => {
  console.error('FATAL:', err);
  closePool().then(() => process.exit(1));
});