← back to Wolfgordon Crawl

Wolf Gordon Mac2-local crawl: parser updated for current Craft-CMS markup (colorways via aria-label, signed CDN images, dt/dd specs), local dw_unified socket

6235ec221470b2c0b420ce16a9bd6b9e6d82a31c · 2026-06-19 09:07:54 -0700 · Steve

Files touched

• A .gitignore
• A node_modules
• A package.json
• A scrape-wolfgordon.js
• A scraper-utils.js

Diff

commit 6235ec221470b2c0b420ce16a9bd6b9e6d82a31c
Author: Steve <steve@designerwallcoverings.com>
Date:   Fri Jun 19 09:07:54 2026 -0700

    Wolf Gordon Mac2-local crawl: parser updated for current Craft-CMS markup (colorways via aria-label, signed CDN images, dt/dd specs), local dw_unified socket
---
 .gitignore           |   5 +
 node_modules         |   1 +
 package.json         |   8 ++
 scrape-wolfgordon.js | 327 +++++++++++++++++++++++++++++++++++++++++++++++++++
 scraper-utils.js     |  63 ++++++++++
 5 files changed, 404 insertions(+)

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ff2422c
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
diff --git a/node_modules b/node_modules
new file mode 120000
index 0000000..8ee9367
--- /dev/null
+++ b/node_modules
@@ -0,0 +1 @@
+/Users/stevestudio2/kamatera-mirror/DW-Agents/vendor-scrapers/node_modules
\ No newline at end of file
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..70497de
--- /dev/null
+++ b/package.json
@@ -0,0 +1,8 @@
+{
+  "name": "wolfgordon-crawl",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Mac2-local Wolf Gordon full catalog crawl into local dw_unified.wolf_gordon_catalog (upsert on mfr_sku, no new SKUs).",
+  "scripts": { "crawl": "node scrape-wolfgordon.js" },
+  "dependencies": { "pg": "*" }
+}
diff --git a/scrape-wolfgordon.js b/scrape-wolfgordon.js
new file mode 100644
index 0000000..2bbcb64
--- /dev/null
+++ b/scrape-wolfgordon.js
@@ -0,0 +1,327 @@
+#!/usr/bin/env node
+// ============================================================================
+// Wolf Gordon Scraper (Commercial Wallcovering)
+// ============================================================================
+// Strategy:
+//   1. Fetch sitemap index for all product sitemap pages
+//   2. Filter for /products/wallcovering/ URLs
+//   3. Also scrape the main /wallcovering page for featured products
+//   4. Parse each product page HTML: data-sku, colorway, collection,
+//      colorway thumbnails (each colorway = separate SKU)
+//
+// Target table: wolf_gordon_catalog
+// Agent: Wade | Port: 9658 | PM2: wolfgordon-agent
+// ============================================================================
+
+const https = require('https');
+const http = require('http');
+const { pool, upsertProduct, wait, getCatalogCount, closePool } = require('./scraper-utils');
+
+const TABLE = 'wolf_gordon_catalog';
+const BASE_URL = 'https://www.wolfgordon.com';
+
+let totalInserted = 0;
+let totalErrors = 0;
+let totalSkipped = 0;
+
+function fetchPage(url, maxRedirects = 5) {
+  return new Promise((resolve, reject) => {
+    const client = url.startsWith('https') ? https : http;
+    client.get(url, {
+      timeout: 25000,
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'en-US,en;q=0.9',
+      }
+    }, (res) => {
+      if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+        let loc = res.headers.location;
+        if (loc.startsWith('/')) loc = `${BASE_URL}${loc}`;
+        return fetchPage(loc, maxRedirects - 1).then(resolve).catch(reject);
+      }
+      let data = '';
+      res.on('data', chunk => data += chunk);
+      res.on('end', () => resolve({ html: data, status: res.statusCode }));
+      res.on('error', reject);
+    }).on('error', reject).on('timeout', function() { this.destroy(); reject(new Error('timeout')); });
+  });
+}
+
+function fetchXml(url) {
+  return new Promise((resolve, reject) => {
+    https.get(url, { timeout: 15000 }, (res) => {
+      if (res.statusCode === 301 || res.statusCode === 302) {
+        return fetchXml(res.headers.location).then(resolve).catch(reject);
+      }
+      let data = '';
+      res.on('data', chunk => data += chunk);
+      res.on('end', () => resolve(data));
+      res.on('error', reject);
+    }).on('error', reject);
+  });
+}
+
+// ============================================================================
+// Step 1: Get wallcovering product URLs from sitemaps
+// ============================================================================
+
+async function getProductUrls() {
+  console.log('\n[1] Fetching sitemap index...');
+  const indexXml = await fetchXml(`${BASE_URL}/sitemap.xml`);
+  
+  // Extract all product sitemap URLs
+  const sitemapUrls = [];
+  const sitemapRegex = /<loc>(https:\/\/www\.wolfgordon\.com\/sitemaps-1-section-products[^<]*)<\/loc>/g;
+  let m;
+  while ((m = sitemapRegex.exec(indexXml)) !== null) {
+    sitemapUrls.push(m[1]);
+  }
+  console.log(`  Found ${sitemapUrls.length} product sitemap pages`);
+  
+  // Fetch each sitemap and extract wallcovering URLs
+  const productUrls = new Set();
+  
+  for (let i = 0; i < sitemapUrls.length; i++) {
+    try {
+      const xml = await fetchXml(sitemapUrls[i]);
+      const locRegex = /<loc>(https:\/\/www\.wolfgordon\.com\/products\/wallcovering\/[^<]+)<\/loc>/g;
+      let m2;
+      while ((m2 = locRegex.exec(xml)) !== null) {
+        productUrls.add(m2[1]);
+      }
+      if ((i + 1) % 5 === 0) console.log(`  Sitemap ${i + 1}/${sitemapUrls.length}: ${productUrls.size} wallcovering URLs`);
+      await wait(200, 100);
+    } catch (err) {
+      console.error(`  Error fetching sitemap ${i}: ${err.message}`);
+    }
+  }
+  
+  // Also get URLs from the main /wallcovering page
+  console.log('  Checking main wallcovering page...');
+  try {
+    const { html } = await fetchPage(`${BASE_URL}/wallcovering`);
+    const linkRegex = /href="(https:\/\/www\.wolfgordon\.com\/products\/wallcovering\/[^"]+)"/gi;
+    let m3;
+    while ((m3 = linkRegex.exec(html)) !== null) {
+      const url = m3[1].split('?')[0]; // Remove query params like preview tokens
+      productUrls.add(url);
+    }
+  } catch (err) {
+    console.error(`  Main page error: ${err.message}`);
+  }
+  
+  console.log(`  Total unique wallcovering product URLs: ${productUrls.size}`);
+  return [...productUrls];
+}
+
+// ============================================================================
+// Step 2: Parse product pages
+// ============================================================================
+
+// Decode the handful of HTML entities WG leaks into text fields.
+function deent(s) {
+  if (!s) return s;
+  return s.replace(/&amp;/g, '&').replace(/&#39;/g, "'").replace(/&quot;/g, '"')
+          .replace(/&rsquo;/g, "’").replace(/&nbsp;/g, ' ').trim();
+}
+
+// Per-colorway CDN swatch image. WG SKU "MAO 6080" -> .../images/MAO-6080.jpg
+function cdnImage(sku) {
+  return `https://cdn2-optimize.wolfgordon.com/production/images/${sku.toUpperCase().replace(/\s+/g, '-')}.jpg`;
+}
+
+// Updated 2026-06-19 for the current Craft-CMS markup. The old markers
+// (data-balloon, product-info__field-details, product-masthead__main-image)
+// are gone. Colorways now live in `aria-label="SKU - Color"` swatch links,
+// specs in a <dt>/<dd> list, pattern name in the ld+json `name`.
+function parseProductPage(html, url) {
+  const products = [];
+
+  const mainSkuMatch = html.match(/data-sku="([^"]+)"/);
+  if (!mainSkuMatch) return products;
+  const mainSku = mainSkuMatch[1].trim();
+
+  // Pattern name from ld+json name: "MAO 6080 - Macao - Primavera | Wallcovering"
+  let productName = null;
+  const nameMatch = html.match(/"name":\s*"([^"]+?)\s*\|\s*Wallcovering"/);
+  if (nameMatch) {
+    const parts = deent(nameMatch[1]).split(' - ').map(s => s.trim());
+    productName = parts.length >= 3 ? parts[1] : parts[0];
+  }
+
+  // Spec fields from the <dt>/<dd> list.
+  const specs = {};
+  const dtdd = html.matchAll(/<dt[^>]*>([\s\S]*?)<\/dt>\s*<dd[^>]*>([\s\S]*?)<\/dd>/g);
+  for (const m of dtdd) {
+    const k = deent(m[1].replace(/<[^>]+>/g, '')).replace(/:$/, '').toLowerCase();
+    const v = deent(m[2].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' '));
+    if (k && v && !specs[k]) specs[k] = v;
+  }
+  const collection = specs['collection'] || null;
+  const material   = specs['material'] || null;
+  const width      = specs['width'] || specs['roll width'] || null;
+  const repeatV    = specs['vertical repeat'] || specs['repeat'] || null;
+  const repeatH    = specs['horizontal repeat'] || null;
+  const matchType  = specs['match'] || null;
+  const fire       = specs['fire rating'] || specs['flammability'] || specs['flame spread'] || null;
+  if (!productName && specs['pattern']) productName = specs['pattern'];
+
+  // Map SKU(dash) -> SIGNED swatch image URL scraped from the page.
+  // The unsigned base URL 401s; the page's signed imgix URLs are public + 200.
+  const imgMap = {};
+  const imgRe = /https:\/\/cdn2-optimize\.wolfgordon\.com\/production\/images\/([A-Z0-9-]+)\.jpg\?[^"' )]+/g;
+  let im;
+  while ((im = imgRe.exec(html)) !== null) {
+    const key = im[1].toUpperCase();
+    if (!imgMap[key]) imgMap[key] = im[0].replace(/&amp;/g, '&');
+  }
+
+  // All colorways from swatch aria-labels: "MAO 6080 - Primavera"
+  const seen = new Set();
+  const colorways = [];
+  const cwRegex = /aria-label="([A-Z]{2,5}\s?[0-9]{3,6})\s*-\s*([^"]+)"/g;
+  let cw;
+  while ((cw = cwRegex.exec(html)) !== null) {
+    const sku = cw[1].replace(/\s+/g, ' ').trim();
+    if (seen.has(sku)) continue;
+    seen.add(sku);
+    colorways.push({ sku, color: deent(cw[2]) });
+  }
+  if (!seen.has(mainSku)) colorways.push({ sku: mainSku, color: specs['colorway'] || null });
+
+  // Derive each colorway's own page URL by swapping the trailing SKU slug.
+  const slug = s => s.toLowerCase().replace(/\s+/g, '-');
+  const base = url.replace(/[?#].*$/, '');
+  const mainSlug = slug(mainSku);
+
+  for (const c of colorways) {
+    let purl = base;
+    if (base.toLowerCase().endsWith(mainSlug)) {
+      purl = base.slice(0, base.length - mainSlug.length) + slug(c.sku);
+    }
+    const img = imgMap[c.sku.toUpperCase().replace(/\s+/g, '-')] || cdnImage(c.sku);
+    products.push({
+      mfr_sku: c.sku,
+      pattern_name: productName,
+      color_name: c.color || null,
+      collection,
+      product_type: 'wallcovering',
+      material,
+      width: width || undefined,
+      repeat_v: repeatV || undefined,
+      repeat_h: repeatH || undefined,
+      match_type: matchType || undefined,
+      fire_rating: fire || undefined,
+      image_url: img,
+      all_images: JSON.stringify([img]),
+      product_url: purl,
+      in_stock: true,
+    });
+  }
+
+  return products;
+}
+
+// ============================================================================
+// Step 3: Scrape with concurrency
+// ============================================================================
+
+async function scrapeProducts(productUrls) {
+  console.log(`\n[2] Scraping ${productUrls.length} product pages via HTTP...`);
+  const concurrency = 3;
+  let index = 0;
+  
+  async function worker() {
+    while (index < productUrls.length) {
+      const i = index++;
+      const url = productUrls[i];
+      try {
+        const { html, status } = await fetchPage(url);
+        if (status !== 200 || html.length < 2000) { totalSkipped++; continue; }
+        
+        const products = parseProductPage(html, url);
+        if (products.length === 0) { totalSkipped++; continue; }
+        
+        for (const product of products) {
+          if (!product.mfr_sku) continue;
+          const id = await upsertProduct(TABLE, product);
+          if (id) {
+            totalInserted++;
+            if (totalInserted <= 8) {
+              console.log(`  + ${product.mfr_sku.padEnd(15)} | ${(product.pattern_name || '').padEnd(20)} | ${(product.color_name || '').padEnd(12)} | ${product.collection || 'N/A'}`);
+            }
+          } else { totalSkipped++; }
+        }
+        
+        if ((i + 1) % 25 === 0) console.log(`  Progress: ${i + 1}/${productUrls.length} pages (${totalInserted} SKUs inserted)`);
+        await wait(400, 200);
+      } catch (err) {
+        totalErrors++;
+        if (totalErrors <= 5) console.error(`  Error: ${url} - ${err.message}`);
+        await wait(800, 400);
+      }
+    }
+  }
+  
+  const workers = [];
+  for (let w = 0; w < concurrency; w++) workers.push(worker());
+  await Promise.all(workers);
+}
+
+// ============================================================================
+// MAIN
+// ============================================================================
+
+async function main() {
+  console.log('='.repeat(70));
+  console.log('  WOLF GORDON SCRAPER (Commercial Wallcovering)');
+  console.log('='.repeat(70));
+  console.log(`  Site: ${BASE_URL}`);
+  console.log(`  Table: ${TABLE}`);
+  console.log(`  Agent: Wade | Port: 9658`);
+  console.log('='.repeat(70));
+  
+  const startTime = Date.now();
+  
+  await pool.query(`
+    CREATE TABLE IF NOT EXISTS ${TABLE} (
+      id SERIAL PRIMARY KEY, mfr_sku TEXT UNIQUE, dw_sku TEXT,
+      pattern_name TEXT, color_name TEXT, collection TEXT, product_type TEXT,
+      width TEXT, length TEXT, repeat_v TEXT, repeat_h TEXT, material TEXT,
+      price_retail NUMERIC(10,2), price_trade NUMERIC(10,2),
+      image_url TEXT, product_url TEXT,
+      in_stock BOOLEAN DEFAULT TRUE, discontinued BOOLEAN DEFAULT FALSE,
+      shopify_product_id TEXT, last_scraped TIMESTAMPTZ,
+      created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW()
+    );
+  `);
+  console.log(`  Table "${TABLE}" ready.`);
+  
+  const productUrls = await getProductUrls();
+  await scrapeProducts(productUrls);
+  
+  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+  const count = await getCatalogCount(TABLE);
+  
+  console.log('\n' + '='.repeat(70));
+  console.log('  WOLF GORDON SCRAPE COMPLETE');
+  console.log('='.repeat(70));
+  console.log(`  Products (SKUs) inserted/updated: ${totalInserted}`);
+  console.log(`  Skipped: ${totalSkipped}`);
+  console.log(`  Errors: ${totalErrors}`);
+  console.log(`  Total in ${TABLE}: ${count}`);
+  console.log(`  Duration: ${elapsed}s`);
+  console.log('='.repeat(70));
+  
+  await closePool();
+}
+
+if (require.main === module) {
+  main().catch(err => {
+    console.error('FATAL:', err);
+    closePool().then(() => process.exit(1));
+  });
+}
+
+module.exports = { parseProductPage, getProductUrls };
diff --git a/scraper-utils.js b/scraper-utils.js
new file mode 100644
index 0000000..d40f312
--- /dev/null
+++ b/scraper-utils.js
@@ -0,0 +1,63 @@
+// ============================================================================
+// Minimal Mac2-local scraper utils for the Wolf Gordon crawl.
+// Self-contained (only `pg`, no browser) — scrape-wolfgordon.js is pure HTTP.
+// Targets the LOCAL dw_unified over the unix socket (peer auth as the OS user),
+// the same DB the existing 5,377 wolf_gordon_catalog rows live in.
+// Prod (Kamatera) sync is a separate, Steve-gated step — NOT this script.
+// ============================================================================
+
+const { Pool } = require('pg');
+
+// Unix-socket / peer auth — mirrors `psql -d dw_unified` working on Mac2.
+// Override with DATABASE_URL if ever needed.
+const pool = process.env.DATABASE_URL
+  ? new Pool({ connectionString: process.env.DATABASE_URL, max: 3 })
+  : new Pool({ host: '/tmp', database: 'dw_unified', max: 3 });
+
+async function upsertProduct(table, product) {
+  if (!product.mfr_sku) return null;
+
+  const fields = Object.keys(product).filter(k => product[k] !== undefined && product[k] !== null);
+  const values = fields.map(k => product[k]);
+  const placeholders = fields.map((_, i) => `$${i + 1}`);
+  const updates = fields.filter(f => f !== 'mfr_sku').map(f => `${f} = EXCLUDED.${f}`);
+
+  const sql = `
+    INSERT INTO ${table} (${fields.join(', ')})
+    VALUES (${placeholders.join(', ')})
+    ON CONFLICT (mfr_sku) DO UPDATE SET
+      ${updates.join(', ')},
+      updated_at = NOW(),
+      last_scraped = NOW()
+    RETURNING id
+  `;
+
+  try {
+    const res = await pool.query(sql, values);
+    return res.rows[0]?.id;
+  } catch (err) {
+    if (err.message.includes('duplicate key')) return null;
+    console.error(`  DB error for SKU ${product.mfr_sku}: ${err.message}`);
+    return null;
+  }
+}
+
+function wait(base = 300, jitter = 150) {
+  const ms = base + Math.floor(Math.random() * jitter);
+  return new Promise(r => setTimeout(r, ms));
+}
+
+async function getCatalogCount(table) {
+  try {
+    const res = await pool.query(`SELECT COUNT(*)::int AS c FROM ${table}`);
+    return res.rows[0].c;
+  } catch {
+    return 0;
+  }
+}
+
+async function closePool() {
+  await pool.end();
+}
+
+module.exports = { pool, upsertProduct, wait, getCatalogCount, closePool };

(oldest)  ·  back to Wolfgordon Crawl  ·  Fix: send User-Agent on sitemap fetch (WG 403s UA-less reque 946e786 →