← back to Wolfgordon Crawl
scrape-wolfgordon.js
356 lines
#!/usr/bin/env node
// ============================================================================
// Wolf Gordon Scraper (Commercial Wallcovering)
// ============================================================================
// Strategy:
// 1. Fetch sitemap index for all product sitemap pages
// 2. Filter for /products/wallcovering/ URLs
// 3. Also scrape the main /wallcovering page for featured products
// 4. Parse each product page HTML: data-sku, colorway, collection,
// colorway thumbnails (each colorway = separate SKU)
//
// Target table: wolf_gordon_catalog
// Agent: Wade | Port: 9658 | PM2: wolfgordon-agent
// ============================================================================
const https = require('https');
const http = require('http');
const { pool, upsertProduct, wait, getCatalogCount, closePool } = require('./scraper-utils');
const TABLE = 'wolf_gordon_catalog';
const BASE_URL = 'https://www.wolfgordon.com';
let totalInserted = 0;
let totalErrors = 0;
let totalSkipped = 0;
function fetchPage(url, maxRedirects = 5) {
return new Promise((resolve, reject) => {
const client = url.startsWith('https') ? https : http;
client.get(url, {
timeout: 25000,
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
}
}, (res) => {
if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
let loc = res.headers.location;
if (loc.startsWith('/')) loc = `${BASE_URL}${loc}`;
return fetchPage(loc, maxRedirects - 1).then(resolve).catch(reject);
}
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => resolve({ html: data, status: res.statusCode }));
res.on('error', reject);
}).on('error', reject).on('timeout', function() { this.destroy(); reject(new Error('timeout')); });
});
}
function fetchXml(url) {
return new Promise((resolve, reject) => {
// WG returns 403 to UA-less requests — a UA is mandatory on the sitemaps too.
https.get(url, { timeout: 15000, headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'application/xml,text/xml,*/*;q=0.8',
} }, (res) => {
if (res.statusCode === 301 || res.statusCode === 302) {
return fetchXml(res.headers.location).then(resolve).catch(reject);
}
let data = '';
res.on('data', chunk => data += chunk);
res.on('end', () => resolve(data));
res.on('error', reject);
}).on('error', reject);
});
}
// ============================================================================
// Step 1: Get wallcovering product URLs from sitemaps
// ============================================================================
async function getProductUrls() {
console.log('\n[1] Fetching sitemap index...');
const indexXml = await fetchXml(`${BASE_URL}/sitemap.xml`);
// Extract all product sitemap URLs
const sitemapUrls = [];
const sitemapRegex = /<loc>(https:\/\/www\.wolfgordon\.com\/sitemaps-1-section-products[^<]*)<\/loc>/g;
let m;
while ((m = sitemapRegex.exec(indexXml)) !== null) {
sitemapUrls.push(m[1]);
}
console.log(` Found ${sitemapUrls.length} product sitemap pages`);
// Fetch each sitemap and extract wallcovering URLs
const productUrls = new Set();
for (let i = 0; i < sitemapUrls.length; i++) {
try {
const xml = await fetchXml(sitemapUrls[i]);
const locRegex = /<loc>(https:\/\/www\.wolfgordon\.com\/products\/wallcovering\/[^<]+)<\/loc>/g;
let m2;
while ((m2 = locRegex.exec(xml)) !== null) {
productUrls.add(m2[1]);
}
if ((i + 1) % 5 === 0) console.log(` Sitemap ${i + 1}/${sitemapUrls.length}: ${productUrls.size} wallcovering URLs`);
await wait(200, 100);
} catch (err) {
console.error(` Error fetching sitemap ${i}: ${err.message}`);
}
}
// Also get URLs from the main /wallcovering page
console.log(' Checking main wallcovering page...');
try {
const { html } = await fetchPage(`${BASE_URL}/wallcovering`);
const linkRegex = /href="(https:\/\/www\.wolfgordon\.com\/products\/wallcovering\/[^"]+)"/gi;
let m3;
while ((m3 = linkRegex.exec(html)) !== null) {
const url = m3[1].split('?')[0]; // Remove query params like preview tokens
productUrls.add(url);
}
} catch (err) {
console.error(` Main page error: ${err.message}`);
}
console.log(` Total unique wallcovering product URLs: ${productUrls.size}`);
return [...productUrls];
}
// ============================================================================
// Step 2: Parse product pages
// ============================================================================
// Decode the handful of HTML entities WG leaks into text fields.
function deent(s) {
if (!s) return s;
return s.replace(/&/g, '&').replace(/'/g, "'").replace(/"/g, '"')
.replace(/’/g, "’").replace(/ /g, ' ').trim();
}
// Per-colorway CDN swatch image. WG SKU "MAO 6080" -> .../images/MAO-6080.jpg
function cdnImage(sku) {
return `https://cdn2-optimize.wolfgordon.com/production/images/${sku.toUpperCase().replace(/\s+/g, '-')}.jpg`;
}
// Updated 2026-06-19 for the current Craft-CMS markup. The old markers
// (data-balloon, product-info__field-details, product-masthead__main-image)
// are gone. Colorways now live in `aria-label="SKU - Color"` swatch links,
// specs in a <dt>/<dd> list, pattern name in the ld+json `name`.
function parseProductPage(html, url) {
const products = [];
const mainSkuMatch = html.match(/data-sku="([^"]+)"/);
if (!mainSkuMatch) return products;
const mainSku = mainSkuMatch[1].trim();
// Pattern name from ld+json name: "MAO 6080 - Macao - Primavera | Wallcovering"
let productName = null;
const nameMatch = html.match(/"name":\s*"([^"]+?)\s*\|\s*Wallcovering"/);
if (nameMatch) {
const parts = deent(nameMatch[1]).split(' - ').map(s => s.trim());
productName = parts.length >= 3 ? parts[1] : parts[0];
}
// Spec fields from the <dt>/<dd> list.
const specs = {};
const dtdd = html.matchAll(/<dt[^>]*>([\s\S]*?)<\/dt>\s*<dd[^>]*>([\s\S]*?)<\/dd>/g);
for (const m of dtdd) {
const k = deent(m[1].replace(/<[^>]+>/g, '')).replace(/:$/, '').toLowerCase();
const v = deent(m[2].replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' '));
if (k && v && !specs[k]) specs[k] = v;
}
const collection = specs['collection'] || null;
const material = specs['material'] || null;
const width = specs['width'] || specs['roll width'] || null;
const repeatV = specs['vertical repeat'] || specs['repeat'] || null;
const repeatH = specs['horizontal repeat'] || null;
const matchType = specs['match'] || null;
const fire = specs['fire rating'] || specs['flammability'] || specs['flame spread'] || null;
if (!productName && specs['pattern']) productName = specs['pattern'];
// Map SKU(dash) -> SIGNED swatch image URL scraped from the page.
// The unsigned base URL 401s; the page's signed imgix URLs are public + 200.
// WG serves signed swatch images under TWO CDN paths: /production/images/<SKU>
// and /production/product/swatches/<sku>. Capture both; key by alnum-only SKU.
const imgMap = {};
const nk = s => s.toUpperCase().replace(/[^A-Z0-9]/g, '');
const imgRe = /https:\/\/cdn2-optimize\.wolfgordon\.com\/production\/(?:images|product\/swatches)\/([A-Za-z0-9_-]+)\.jpg\?[^"' )]+/g;
let im;
while ((im = imgRe.exec(html)) !== null) {
const key = nk(im[1]);
if (!imgMap[key]) imgMap[key] = im[0].replace(/&/g, '&');
}
// ROOM / INSTALLATION shot — shared per pattern (not per colorway). Lives under
// production/product/installation/<...>/<pattern>.jpg with a signed query string.
// ROOT-CAUSE FIX (2026-06-19): the old parser captured ONLY the swatch, so
// all_images was always a single image and products published with no room
// shot. Capture every distinct installation image and prepend it to all_images
// for each colorway, honoring the standing rule "always pull all data + images".
// De-dup by image PATH (ignore the resolution/signature query), keeping the
// first (largest in srcset order) signed URL per distinct installation file.
const roomByPath = {};
const roomRe = /https:\/\/cdn2-optimize\.wolfgordon\.com\/production\/product\/installation\/[^"'?\s,)]+\.jpg\?[^"'\s,)]+/g;
let rm;
while ((rm = roomRe.exec(html)) !== null) {
const u = rm[0].replace(/&/g, '&').replace(/,+$/, '');
const path = u.split('?')[0];
if (!roomByPath[path]) roomByPath[path] = u; // first signed URL wins
}
const roomImgs = Object.values(roomByPath);
// All colorways from swatch aria-labels: "MAO 6080 - Primavera"
const seen = new Set();
const colorways = [];
const cwRegex = /aria-label="([A-Z]{2,5}\s?[0-9]{3,6})\s*-\s*([^"]+)"/g;
let cw;
while ((cw = cwRegex.exec(html)) !== null) {
const sku = cw[1].replace(/\s+/g, ' ').trim();
if (seen.has(sku)) continue;
seen.add(sku);
colorways.push({ sku, color: deent(cw[2]) });
}
if (!seen.has(mainSku)) colorways.push({ sku: mainSku, color: specs['colorway'] || null });
// Derive each colorway's own page URL by swapping the trailing SKU slug.
const slug = s => s.toLowerCase().replace(/\s+/g, '-');
const base = url.replace(/[?#].*$/, '');
const mainSlug = slug(mainSku);
for (const c of colorways) {
let purl = base;
if (base.toLowerCase().endsWith(mainSlug)) {
purl = base.slice(0, base.length - mainSlug.length) + slug(c.sku);
}
const img = imgMap[nk(c.sku)] || cdnImage(c.sku);
// all_images = the colorway swatch (primary) + every shared room/installation
// shot. De-dup, swatch first so image_url stays the primary product image.
const allImgs = [img, ...roomImgs.filter(u => u !== img)];
products.push({
mfr_sku: c.sku,
pattern_name: productName,
color_name: c.color || null,
collection,
product_type: 'wallcovering',
material,
width: width || undefined,
repeat_v: repeatV || undefined,
repeat_h: repeatH || undefined,
match_type: matchType || undefined,
fire_rating: fire || undefined,
image_url: img,
all_images: JSON.stringify(allImgs),
product_url: purl,
in_stock: true,
});
}
return products;
}
// ============================================================================
// Step 3: Scrape with concurrency
// ============================================================================
async function scrapeProducts(productUrls) {
console.log(`\n[2] Scraping ${productUrls.length} product pages via HTTP...`);
const concurrency = 3;
let index = 0;
async function worker() {
while (index < productUrls.length) {
const i = index++;
const url = productUrls[i];
try {
const { html, status } = await fetchPage(url);
if (status !== 200 || html.length < 2000) { totalSkipped++; continue; }
const products = parseProductPage(html, url);
if (products.length === 0) { totalSkipped++; continue; }
for (const product of products) {
if (!product.mfr_sku) continue;
const id = await upsertProduct(TABLE, product);
if (id) {
totalInserted++;
if (totalInserted <= 8) {
console.log(` + ${product.mfr_sku.padEnd(15)} | ${(product.pattern_name || '').padEnd(20)} | ${(product.color_name || '').padEnd(12)} | ${product.collection || 'N/A'}`);
}
} else { totalSkipped++; }
}
if ((i + 1) % 25 === 0) console.log(` Progress: ${i + 1}/${productUrls.length} pages (${totalInserted} SKUs inserted)`);
await wait(400, 200);
} catch (err) {
totalErrors++;
if (totalErrors <= 5) console.error(` Error: ${url} - ${err.message}`);
await wait(800, 400);
}
}
}
const workers = [];
for (let w = 0; w < concurrency; w++) workers.push(worker());
await Promise.all(workers);
}
// ============================================================================
// MAIN
// ============================================================================
async function main() {
console.log('='.repeat(70));
console.log(' WOLF GORDON SCRAPER (Commercial Wallcovering)');
console.log('='.repeat(70));
console.log(` Site: ${BASE_URL}`);
console.log(` Table: ${TABLE}`);
console.log(` Agent: Wade | Port: 9658`);
console.log('='.repeat(70));
const startTime = Date.now();
await pool.query(`
CREATE TABLE IF NOT EXISTS ${TABLE} (
id SERIAL PRIMARY KEY, mfr_sku TEXT UNIQUE, dw_sku TEXT,
pattern_name TEXT, color_name TEXT, collection TEXT, product_type TEXT,
width TEXT, length TEXT, repeat_v TEXT, repeat_h TEXT, material TEXT,
price_retail NUMERIC(10,2), price_trade NUMERIC(10,2),
image_url TEXT, product_url TEXT,
in_stock BOOLEAN DEFAULT TRUE, discontinued BOOLEAN DEFAULT FALSE,
shopify_product_id TEXT, last_scraped TIMESTAMPTZ,
created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW()
);
`);
console.log(` Table "${TABLE}" ready.`);
const productUrls = await getProductUrls();
await scrapeProducts(productUrls);
const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
const count = await getCatalogCount(TABLE);
console.log('\n' + '='.repeat(70));
console.log(' WOLF GORDON SCRAPE COMPLETE');
console.log('='.repeat(70));
console.log(` Products (SKUs) inserted/updated: ${totalInserted}`);
console.log(` Skipped: ${totalSkipped}`);
console.log(` Errors: ${totalErrors}`);
console.log(` Total in ${TABLE}: ${count}`);
console.log(` Duration: ${elapsed}s`);
console.log('='.repeat(70));
await closePool();
}
if (require.main === module) {
main().catch(err => {
console.error('FATAL:', err);
closePool().then(() => process.exit(1));
});
}
module.exports = { parseProductPage, getProductUrls };