← back to Dw Vendor Scrapers Local

scrape-hollywood.js

347 lines

#!/usr/bin/env node
// ============================================================================
// Hollywood Wallcoverings Scraper
// ============================================================================
// Strategy: Shopify JSON API from designerwallcoverings.com
//   - hollywoodwallcoverings.com redirects to designerwallcoverings.com
//   - Single collection: hollywood-wallcovering-collection
//   - ~5000+ products, paginate products.json?limit=250&page=N
//   - Extract: SKU, pattern, color, collection, material, price, image, URL
//
// Target table: hollywood_catalog
// ============================================================================

const { pool, upsertProduct, wait, closePool } = require('./scraper-utils');
const https = require('https');

const TABLE = 'hollywood_catalog';
const SHOP_BASE = 'https://www.designerwallcoverings.com';
const COLLECTION_SLUG = 'hollywood-wallcovering-collection';

let totalInserted = 0;
let totalErrors = 0;
let totalSkipped = 0;

// ============================================================================
// HTTP helper
// ============================================================================

const httpsAgent = new https.Agent({ rejectUnauthorized: false });

function httpGetJson(url) {
  return new Promise((resolve, reject) => {
    const urlObj = new URL(url);
    const options = {
      hostname: urlObj.hostname,
      path: urlObj.pathname + urlObj.search,
      method: 'GET',
      agent: httpsAgent,
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
        'Accept': 'application/json',
      },
      timeout: 30000,
    };

    const req = https.request(options, (res) => {
      if (res.statusCode === 301 || res.statusCode === 302) {
        return httpGetJson(res.headers.location).then(resolve).catch(reject);
      }
      let data = '';
      res.on('data', chunk => data += chunk);
      res.on('end', () => {
        try {
          resolve(JSON.parse(data));
        } catch (e) {
          reject(new Error(`JSON parse error at ${url}: ${e.message}`));
        }
      });
    });

    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
    req.end();
  });
}

// ============================================================================
// Parse a Shopify product into catalog format
// ============================================================================

function parseShopifyProduct(product) {
  const title = product.title || '';
  const tags = product.tags || [];
  const handle = product.handle || '';

  // Title format: "Abbingdon Type II Vinyl  Wallpaper | Hollywood Wallcoverings"
  // or: "Pattern Name - Color | Hollywood Wallcoverings"
  let pattern_name = null;
  let color_name = null;

  // Remove vendor suffix
  let cleanTitle = title.replace(/\s*\|\s*Hollywood\s+Wallcoverings\s*/i, '').trim();

  // Try "Pattern - Color" format
  const dashMatch = cleanTitle.match(/^(.+?)\s*-\s*(.+?)(?:\s+(?:Type\s+II|Vinyl|Wallpaper|Wallcovering))?$/i);
  if (dashMatch) {
    pattern_name = dashMatch[1].trim();
    color_name = dashMatch[2].trim();
  } else {
    // Use full title as pattern name, clean up type suffixes
    pattern_name = cleanTitle
      .replace(/\s+Type\s+II\s+Vinyl\s+Wallpaper/i, '')
      .replace(/\s+Type\s+II\s+Vinyl\s+Wallcovering/i, '')
      .replace(/\s+Wallpaper/i, '')
      .replace(/\s+Wallcovering/i, '')
      .trim();
  }

  // Try to extract color from tags
  if (!color_name) {
    const colorTags = tags.filter(t =>
      !t.match(/Hollywood|Wallcovering|Wallpaper|vinyl|display_variant|Type\s*II/i)
      && !t.match(/^\d/)
      && t.length > 2
      && t.length < 30
    );
    // Use the tag matching the pattern name to find the color
    const patternTag = tags.find(t => t === pattern_name);
    if (patternTag) {
      // Pattern tag found, color is the next non-matching descriptive tag
      const colorCandidates = colorTags.filter(t => t !== pattern_name);
      if (colorCandidates.length > 0) color_name = colorCandidates[0];
    }
  }

  // Variants - get SKU and price
  const variants = product.variants || [];
  let mfr_sku = null;
  let price_retail = null;

  // Prefer non-sample variant
  const fullVariant = variants.find(v => !v.sku?.toLowerCase().includes('sample'));
  const sampleVariant = variants.find(v => v.sku?.toLowerCase().includes('sample'));

  if (fullVariant) {
    mfr_sku = fullVariant.sku;
    price_retail = parseFloat(fullVariant.price) || null;
  } else if (sampleVariant) {
    mfr_sku = sampleVariant.sku.replace(/-[Ss]ample$/i, '');
    price_retail = null;
  }

  if (!mfr_sku) mfr_sku = handle;

  // Extract material from tags
  let material = null;
  if (tags.some(t => t.match(/vinyl/i))) material = 'Vinyl';
  else if (tags.some(t => t.match(/100%\s*vinyl/i))) material = '100% Vinyl';
  else if (tags.some(t => t.match(/Type\s*II/i))) material = 'Type II Vinyl';

  // Product type
  const product_type = product.product_type || 'Commercial Wallcoverings';

  // Collection detection from tags
  let collection = 'Hollywood Wallcoverings';
  // Some tags might indicate sub-collections
  const collTag = tags.find(t => t.match(/collection/i) && !t.match(/Hollywood/i));
  if (collTag) collection = collTag;

  // Images — capture the FULL array, not just the hero.
  // The Shopify products.json feed exposes product.images[] (often 8+ per SKU);
  // the old code kept only images[0] and discarded the rest. We now persist all
  // full-resolution srcs in all_images (JSON array string) and keep image_url as
  // the first/hero for backward compatibility.
  const imageSrcs = (product.images || [])
    .map(i => i && i.src)
    .filter(Boolean);
  const image_url = imageSrcs[0] || null;
  const all_images = imageSrcs.length ? JSON.stringify(imageSrcs) : null;

  // Product URL
  const product_url = `${SHOP_BASE}/products/${handle}`;

  // ALL DATA — capture additional feed fields the old parser dropped, into a
  // specs jsonb blob so nothing useful from the source is thrown away.
  const allVariantSkus = variants
    .map(v => v && v.sku)
    .filter(Boolean);
  const specs = {
    shopify_product_id: product.id ? String(product.id) : null,
    handle,
    title,
    body_html: product.body_html || null,        // spec/description text
    vendor: product.vendor || null,
    product_type: product.product_type || null,
    published_at: product.published_at || null,
    created_at: product.created_at || null,
    updated_at: product.updated_at || null,
    tags,
    options: product.options || null,             // e.g. Title / sample-vs-roll
    variant_skus: allVariantSkus,                 // all variant SKUs (roll + sample)
    image_count: imageSrcs.length,
    image_srcs: imageSrcs,                        // redundant w/ all_images, kept in specs for completeness
  };

  return {
    mfr_sku,
    pattern_name,
    color_name,
    collection,
    product_type,
    material,
    price_retail,
    image_url,
    all_images,
    product_url,
    shopify_product_id: product.id ? String(product.id) : null,
    specs: JSON.stringify(specs),
  };
}

// ============================================================================
// MAIN
// ============================================================================

async function main() {
  console.log('='.repeat(70));
  console.log('  HOLLYWOOD WALLCOVERINGS SCRAPER');
  console.log('='.repeat(70));
  console.log(`  Source: Shopify JSON API (${SHOP_BASE})`);
  console.log(`  Collection: ${COLLECTION_SLUG}`);
  console.log(`  Target table: ${TABLE}`);
  console.log('='.repeat(70));

  const startTime = Date.now();

  // Ensure table exists
  try {
    await pool.query(`SELECT 1 FROM ${TABLE} LIMIT 1`);
    console.log(`\n  Table "${TABLE}" exists.`);
  } catch (err) {
    console.log(`\n  Table "${TABLE}" does not exist, creating...`);
    await pool.query(`
      CREATE TABLE IF NOT EXISTS ${TABLE} (
        id SERIAL PRIMARY KEY,
        mfr_sku TEXT UNIQUE,
        dw_sku TEXT,
        pattern_name TEXT,
        color_name TEXT,
        collection TEXT,
        product_type TEXT,
        width TEXT,
        length TEXT,
        repeat_v TEXT,
        repeat_h TEXT,
        material TEXT,
        price_retail NUMERIC(10,2),
        price_trade NUMERIC(10,2),
        image_url TEXT,
        product_url TEXT,
        in_stock BOOLEAN DEFAULT TRUE,
        discontinued BOOLEAN DEFAULT FALSE,
        shopify_product_id TEXT,
        last_scraped TIMESTAMPTZ,
        created_at TIMESTAMPTZ DEFAULT NOW(),
        updated_at TIMESTAMPTZ DEFAULT NOW()
      );
    `);
    console.log(`  Table created.`);
  }

  // Ensure the columns we now write exist (idempotent — safe on every run).
  // all_images stores the full image array (JSON), specs stores the rest of the
  // feed (body_html, all variant SKUs, options, etc.) so no source data is lost.
  await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS all_images TEXT`);
  await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS shopify_product_id TEXT`);
  await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS specs JSONB`);

  // Paginate through collection
  let page = 1;
  let grandTotal = 0;
  const maxPages = 60; // Safety limit (collection ~3,300 products / 250 = ~14 pages)

  while (page <= maxPages) {
    try {
      const url = `${SHOP_BASE}/collections/${COLLECTION_SLUG}/products.json?limit=250&page=${page}`;
      const data = await httpGetJson(url);
      const products = data.products || [];

      if (products.length === 0) break;

      for (const product of products) {
        const parsed = parseShopifyProduct(product);
        if (!parsed || !parsed.mfr_sku) {
          totalSkipped++;
          continue;
        }

        try {
          const id = await upsertProduct(TABLE, parsed);
          if (id) totalInserted++;
          else totalSkipped++;
        } catch (err) {
          totalErrors++;
        }
        grandTotal++;
      }

      console.log(`  Page ${page}: ${products.length} products (total: ${grandTotal}, inserted: ${totalInserted})`);

      if (products.length < 250) break;
      page++;
      await wait(500, 300);
    } catch (err) {
      console.error(`  Error on page ${page}: ${err.message}`);
      totalErrors++;
      // Retry once after delay
      await wait(3000, 1000);
      try {
        const url = `${SHOP_BASE}/collections/${COLLECTION_SLUG}/products.json?limit=250&page=${page}`;
        const data = await httpGetJson(url);
        const products = data.products || [];
        if (products.length === 0) break;
        for (const product of products) {
          const parsed = parseShopifyProduct(product);
          if (parsed && parsed.mfr_sku) {
            const id = await upsertProduct(TABLE, parsed);
            if (id) totalInserted++;
            grandTotal++;
          }
        }
        console.log(`  Page ${page} (retry): ${products.length} products`);
      } catch (retryErr) {
        console.error(`  Retry failed for page ${page}, skipping.`);
      }
      page++;
      await wait(1000, 500);
    }
  }

  // Final stats
  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
  let count = 0;
  try {
    const res = await pool.query(`SELECT COUNT(*) as cnt FROM ${TABLE}`);
    count = parseInt(res.rows[0].cnt);
  } catch (_) {}

  console.log('\n' + '='.repeat(70));
  console.log('  SCRAPE COMPLETE - HOLLYWOOD WALLCOVERINGS');
  console.log('='.repeat(70));
  console.log(`  Products processed: ${grandTotal}`);
  console.log(`  Products inserted/updated: ${totalInserted}`);
  console.log(`  Skipped: ${totalSkipped}`);
  console.log(`  Errors: ${totalErrors}`);
  console.log(`  Total in ${TABLE}: ${count}`);
  console.log(`  Duration: ${elapsed}s`);
  console.log('='.repeat(70));

  await closePool();
}

main().catch(err => {
  console.error('FATAL:', err);
  closePool().then(() => process.exit(1));
});