← back to Dw Vendor Scrapers Local
Add recrawl-coordonne.js: additive all_images re-crawl + local scraper mirror
2933b617163bff14dfb228f4c540d1dcf5b45108 · 2026-06-10 11:37:17 -0700 · Steve
Files touched
A .gitignoreA README.mdA recrawl-coordonne.jsA recrawl-osborne.jsA scrape-coordonne.jsA scrape-hollywood.js
Diff
commit 2933b617163bff14dfb228f4c540d1dcf5b45108
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jun 10 11:37:17 2026 -0700
Add recrawl-coordonne.js: additive all_images re-crawl + local scraper mirror
---
.gitignore | 8 ++
README.md | 15 +++
recrawl-coordonne.js | 190 ++++++++++++++++++++++++++++
recrawl-osborne.js | 77 ++++++++++++
scrape-coordonne.js | 288 ++++++++++++++++++++++++++++++++++++++++++
scrape-hollywood.js | 346 +++++++++++++++++++++++++++++++++++++++++++++++++++
6 files changed, 924 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1924158
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..b5080b6
--- /dev/null
+++ b/README.md
@@ -0,0 +1,15 @@
+# dw-vendor-scrapers-local
+
+Local working copy of select Kamatera DW vendor scrapers
+(`/root/DW-Agents/vendor-scrapers/`) for editing + version control on Mac2.
+
+- `scrape-coordonne.js`, `scrape-hollywood.js`, `recrawl-osborne.js` — read-only
+ reference pulls of the deployed Kamatera scrapers.
+- `recrawl-coordonne.js` — NEW (2026-06-10). Additive `all_images` re-crawl for
+ the coordonne onboarding cohort. The original WP-API scraper captured only one
+ hero image; this re-crawls each product PAGE and fills `all_images` + `specs`
+ on existing rows (UPDATE only — never inserts, never publishes, never touches
+ SKUs). Run `--canary 5` first to confirm the gallery selector against live HTML,
+ then `--all --net-new` for the full pass. Deploy + run is Steve-gated.
+
+Local commits only. No push, no deploy.
diff --git a/recrawl-coordonne.js b/recrawl-coordonne.js
new file mode 100644
index 0000000..bc1bb24
--- /dev/null
+++ b/recrawl-coordonne.js
@@ -0,0 +1,190 @@
+#!/usr/bin/env node
+// ============================================================================
+// Coordonne ADDITIVE all_images re-crawl — Codename: Carmen (full-page pass)
+// ============================================================================
+// WHY: the original scrape-coordonne.js (WordPress REST API) captured only a
+// single hero image (yoast og_image[0]) and NO gallery / per-colorway swatch
+// images. Steve's standing rule "always pull all data and images" requires
+// every product to carry its FULL image set (all angles + colorway swatches)
+// before it is import-ready. This driver re-crawls the actual product PAGE for
+// every coordonne row that is missing all_images and fills all_images + specs.
+//
+// SAFETY / ADDITIVE-ONLY:
+// * It NEVER inserts new rows. It only UPDATEs all_images + specs on rows that
+// already exist (matched by the mfr_sku carried out of the DB).
+// * It only touches rows whose all_images is empty AND whose product_url is a
+// real coordonne.com page (the legacy already-on-Shopify cohort has no
+// coordonne.com URL and is skipped — those are dedup'd out at publish anyway).
+// * It does NOT push to Shopify, does NOT publish, does NOT change SKUs.
+//
+// MODES:
+// --canary N re-crawl only the first N rows (default 5) — run this FIRST to
+// confirm the gallery selector matches live HTML before the full run.
+// --all re-crawl every eligible row (the full ~2,857-page pass).
+// --net-new restrict to rows whose mfr_sku is NOT already in shopify_products
+// (skip the legacy dup cohort entirely).
+//
+// Mirrors the harness of the FIXED recrawl-osborne.js (httpGet + worker pool).
+// ============================================================================
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const https = require('https');
+const { pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
+
+const TABLE = 'coordonne_catalog';
+const SITE_BASE = 'https://coordonne.com';
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+const argv = process.argv.slice(2);
+const FULL = argv.includes('--all');
+const NET_NEW_ONLY = argv.includes('--net-new');
+const canaryIdx = argv.indexOf('--canary');
+const CANARY_N = canaryIdx >= 0 ? (parseInt(argv[canaryIdx + 1], 10) || 5) : (FULL ? 0 : 5);
+
+function httpGet(url, maxRedirects = 4) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' } }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
+ res.resume();
+ return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+// Normalize a WP uploads URL to its full-size original by stripping the
+// -<W>x<H> size suffix WooCommerce/WP appends to resized derivatives.
+function toFullSize(u) {
+ return u.replace(/-\d+x\d+(\.(?:jpe?g|png|webp))(\?.*)?$/i, '$1');
+}
+
+// Junk filters — never store logos, icons, payment badges, flags, avatars,
+// placeholders, or sprite assets as product images.
+const JUNK = /(logo|icon|favicon|placeholder|sprite|payment|visa|mastercard|paypal|flag|avatar|loader|spinner|swatch-sprite|woocommerce-placeholder)/i;
+
+// Extract every product image from a coordonne product page.
+// Covers WooCommerce-style galleries (data-large_image / gallery anchors) and
+// any wp-content/uploads <img> on the page, normalized to full size + deduped.
+function extractImagesFromHtml(html) {
+ const found = [];
+ const push = (raw) => {
+ if (!raw) return;
+ let u = raw.trim().replace(/&/g, '&');
+ if (u.startsWith('//')) u = 'https:' + u;
+ if (!/^https?:\/\//i.test(u)) return;
+ if (!/wp-content\/uploads\//i.test(u)) return; // product imagery only
+ if (!/\.(jpe?g|png|webp)(\?.*)?$/i.test(u)) return; // real image files only
+ if (JUNK.test(u)) return;
+ u = toFullSize(u);
+ if (!found.includes(u)) found.push(u);
+ };
+
+ // 1) WooCommerce gallery: data-large_image="FULL.jpg" (the canonical full-res)
+ let m;
+ const reLarge = /data-large_image=["']([^"']+)["']/gi;
+ while ((m = reLarge.exec(html))) push(m[1]);
+
+ // 2) Gallery anchors wrapping the full image: <a href="...uploads...jpg">
+ const reAnchor = /<a[^>]+href=["']([^"']+wp-content\/uploads\/[^"']+\.(?:jpe?g|png|webp)[^"']*)["']/gi;
+ while ((m = reAnchor.exec(html))) push(m[1]);
+
+ // 3) Any <img src / data-src / srcset> pointing at uploads (gallery thumbs,
+ // colorway swatches, lifestyle shots) — normalized to full size + deduped.
+ const reImg = /<img\b[^>]*?\b(?:data-src|data-large_image|src)=["']([^"']+)["'][^>]*>/gi;
+ while ((m = reImg.exec(html))) push(m[1]);
+ const reSrcset = /\bsrcset=["']([^"']+)["']/gi;
+ while ((m = reSrcset.exec(html))) {
+ for (const part of m[1].split(',')) push(part.trim().split(/\s+/)[0]);
+ }
+
+ // 4) og:image as a fallback hero if the gallery yielded nothing.
+ const og = html.match(/<meta[^>]+property=["']og:image["'][^>]+content=["']([^"']+)["']/i);
+ if (og) push(og[1]);
+
+ return found;
+}
+
+// Pull the WooCommerce additional-information / attributes table as a specs map.
+function extractSpecsFromHtml(html) {
+ const specs = {};
+ const tableMatch = html.match(/<table[^>]*class=["'][^"']*woocommerce-product-attributes[^"']*["'][\s\S]*?<\/table>/i);
+ const scope = tableMatch ? tableMatch[0] : html;
+ const reRow = /<tr[^>]*>[\s\S]*?<th[^>]*>([\s\S]*?)<\/th>[\s\S]*?<td[^>]*>([\s\S]*?)<\/td>[\s\S]*?<\/tr>/gi;
+ let m;
+ while ((m = reRow.exec(scope))) {
+ const k = m[1].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
+ const v = m[2].replace(/<[^>]+>/g, '').replace(/\s+/g, ' ').trim();
+ if (k && v && k.length < 60) specs[k] = v;
+ }
+ return specs;
+}
+
+(async () => {
+ // Eligible rows: missing all_images + a real coordonne.com product page.
+ let sql = `
+ SELECT mfr_sku, split_part(product_url,'?',1) AS url
+ FROM ${TABLE}
+ WHERE (all_images IS NULL OR all_images = '' OR all_images = '[]')
+ AND product_url ~ 'coordonne\\.com'
+ `;
+ if (NET_NEW_ONLY) {
+ sql += ` AND NOT EXISTS (SELECT 1 FROM shopify_products s WHERE upper(s.mfr_sku)=upper(${TABLE}.mfr_sku))`;
+ }
+ sql += ` ORDER BY mfr_sku`;
+ if (CANARY_N > 0) sql += ` LIMIT ${CANARY_N}`;
+
+ // Ensure target columns exist (idempotent).
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS all_images TEXT`);
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS specs JSONB`);
+
+ const { rows } = await pool.query(sql);
+ console.log(`Coordonne additive all_images re-crawl — ${rows.length} pages${CANARY_N > 0 ? ` (CANARY/limit ${CANARY_N})` : ' (FULL)'}${NET_NEW_ONLY ? ' [net-new only]' : ''}`);
+
+ let pages = 0, updated = 0, withImgs = 0, withSpecs = 0, noImgs = 0, errs = 0;
+ const concurrency = 3;
+ let idx = 0;
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const { mfr_sku, url } = rows[i];
+ try {
+ const html = await httpGet(url);
+ const images = extractImagesFromHtml(html);
+ const specs = extractSpecsFromHtml(html);
+ pages++;
+ if (!images.length) { noImgs++; await sleep(300); continue; }
+ // ADDITIVE UPDATE — fill all_images (+ keep image_url as hero) + specs.
+ await pool.query(
+ `UPDATE ${TABLE}
+ SET all_images = $1,
+ image_url = COALESCE(NULLIF(image_url,''), $2),
+ specs = COALESCE(specs, '{}'::jsonb) || $3::jsonb,
+ last_scraped = NOW()
+ WHERE mfr_sku = $4`,
+ [JSON.stringify(images), images[0], JSON.stringify(specs), mfr_sku]
+ );
+ updated++; withImgs++;
+ if (Object.keys(specs).length) withSpecs++;
+ if (CANARY_N > 0) {
+ console.log(` ${mfr_sku}: ${images.length} imgs, ${Object.keys(specs).length} specs`);
+ console.log(` ${images.slice(0, 3).join('\n ')}${images.length > 3 ? '\n …' : ''}`);
+ } else if (pages % 50 === 0) {
+ console.log(` ${pages}/${rows.length} | updated ${updated} | w/imgs ${withImgs} | w/specs ${withSpecs} | noImgs ${noImgs} | errs ${errs}`);
+ }
+ await sleep(350);
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0 || CANARY_N > 0) console.error(` err ${mfr_sku} ${url}: ${e.message}`);
+ await sleep(800);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: concurrency }, worker));
+ console.log(`\nDONE. pages=${pages} updated=${updated} with_imgs=${withImgs} with_specs=${withSpecs} no_imgs=${noImgs} errors=${errs}`);
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });
diff --git a/recrawl-osborne.js b/recrawl-osborne.js
new file mode 100644
index 0000000..8623a61
--- /dev/null
+++ b/recrawl-osborne.js
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+// Re-crawl driver: pulls every distinct osborne product_url from the DB and
+// re-runs the FIXED extraction (images + specs) via the deployed scraper's
+// exported functions, then upserts into osborne_catalog. Avoids the flaky
+// browser collection-listing crawl by reusing known product URLs.
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const fs = require('fs');
+const https = require('https');
+const { upsertProduct, pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
+
+// Load extractFromHtml from the deployed scraper without running its main().
+// Write the temp module INSIDE vendor-scrapers so node_modules (dotenv/pg) resolve.
+const EXTRACT_TMP = '/root/DW-Agents/vendor-scrapers/_osb_extract.js';
+let src = fs.readFileSync('/root/DW-Agents/vendor-scrapers/scrape-osborne.js', 'utf8')
+ .replace(/main\(\)\.catch\([\s\S]*$/, 'module.exports={extractFromHtml};\n');
+fs.writeFileSync(EXTRACT_TMP, src);
+const { extractFromHtml } = require(EXTRACT_TMP);
+
+const SITE_BASE = 'https://www.osborneandlittle.com';
+const TABLE = 'osborne_catalog';
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+function httpGet(url, maxRedirects = 4) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' } }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
+ res.resume();
+ return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+(async () => {
+ const { rows } = await pool.query(
+ `SELECT DISTINCT split_part(product_url,'?',1) AS url FROM ${TABLE} WHERE product_url IS NOT NULL AND product_url <> '' ORDER BY 1`
+ );
+ console.log(`Re-crawling ${rows.length} distinct osborne product pages...`);
+ let pages = 0, skus = 0, withImg = 0, withSpecs = 0, errs = 0;
+ const concurrency = 3;
+ let idx = 0;
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const url = rows[i].url;
+ try {
+ const html = await httpGet(url);
+ const products = extractFromHtml(html, url);
+ for (const p of products) {
+ if (!p.mfr_sku) continue;
+ const id = await upsertProduct(TABLE, p);
+ if (id) {
+ skus++;
+ if (p.image_url) withImg++;
+ if (p.specs) withSpecs++;
+ }
+ }
+ pages++;
+ if (pages % 25 === 0) console.log(` ${pages}/${rows.length} pages | ${skus} skus | ${withImg} w/img | ${withSpecs} w/specs | ${errs} errs`);
+ await sleep(350);
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0) console.error(` err at ${url}: ${e.message}`);
+ await sleep(800);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: concurrency }, worker));
+ console.log(`\nDONE. pages=${pages} skus_upserted=${skus} with_img=${withImg} with_specs=${withSpecs} errors=${errs}`);
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });
diff --git a/scrape-coordonne.js b/scrape-coordonne.js
new file mode 100644
index 0000000..04f85d5
--- /dev/null
+++ b/scrape-coordonne.js
@@ -0,0 +1,288 @@
+#!/usr/bin/env node
+// ============================================================================
+// Coordonne Catalog Scraper — Codename: Carmen
+// ============================================================================
+// Strategy: WordPress REST API only (no individual page scraping needed)
+// The WP API provides title, collection, color, image, and product URL
+// Total products: ~2,800+ (cat 98 = repeats, cat 206 = murals)
+// Note: X-WP-Total headers not returned by Cloudflare, so we paginate until empty
+// ============================================================================
+
+const https = require('https');
+const { pool, upsertProduct, wait, getCatalogCount, closePool } = require('./scraper-utils');
+
+const TABLE = 'coordonne_catalog';
+const BASE_URL = 'https://coordonne.com';
+const PER_PAGE = 100;
+
+// Wallpaper categories
+const WALLPAPER_CATS = [
+ { id: 98, name: 'Repeated Patterns', type: 'wallcovering' },
+ { id: 206, name: 'Murals', type: 'mural' },
+];
+
+let totalInserted = 0;
+let totalSkipped = 0;
+let totalErrors = 0;
+
+function fetchJSON(url) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, {
+ timeout: 30000,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
+ 'Accept': 'application/json',
+ }
+ }, (res) => {
+ if (res.statusCode === 301 || res.statusCode === 302) {
+ const loc = res.headers.location || '';
+ return fetchJSON(loc.startsWith('http') ? loc : `${BASE_URL}${loc}`).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', chunk => data += chunk);
+ res.on('end', () => {
+ try {
+ const parsed = JSON.parse(data);
+ resolve(parsed);
+ } catch (e) {
+ reject(new Error(`JSON parse error: ${e.message}`));
+ }
+ });
+ res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('Request timeout')); });
+ });
+}
+
+async function ensureTable() {
+ try {
+ await pool.query(`SELECT 1 FROM ${TABLE} LIMIT 1`);
+ console.log(` Table "${TABLE}" exists.`);
+ } catch (err) {
+ console.log(` Creating table "${TABLE}"...`);
+ await pool.query(`
+ CREATE TABLE IF NOT EXISTS ${TABLE} (
+ id SERIAL PRIMARY KEY, mfr_sku TEXT UNIQUE, dw_sku TEXT,
+ pattern_name TEXT, color_name TEXT, collection TEXT, product_type TEXT,
+ width TEXT, length TEXT, repeat_v TEXT, repeat_h TEXT, material TEXT,
+ price_retail NUMERIC(10,2), price_trade NUMERIC(10,2),
+ image_url TEXT, product_url TEXT,
+ in_stock BOOLEAN DEFAULT TRUE, discontinued BOOLEAN DEFAULT FALSE,
+ shopify_product_id TEXT, last_scraped TIMESTAMPTZ,
+ created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW()
+ );
+ `);
+ console.log(` Table created.`);
+ }
+}
+
+async function fetchTaxonomyMap(endpoint) {
+ const map = {};
+ let page = 1;
+ while (true) {
+ try {
+ const items = await fetchJSON(`${BASE_URL}/wp-json/wp/v2/${endpoint}?per_page=100&page=${page}`);
+ if (!Array.isArray(items) || items.length === 0) break;
+ for (const c of items) {
+ map[c.id] = c.name;
+ }
+ if (items.length < 100) break;
+ page++;
+ await wait(500, 200);
+ } catch (err) {
+ // rest_post_invalid_page_number means we're past the last page
+ if (err.message && err.message.includes('rest_post_invalid_page_number')) break;
+ console.error(` Error fetching ${endpoint} page ${page}: ${err.message}`);
+ break;
+ }
+ }
+ return map;
+}
+
+function extractProductData(item, collectionMap, colorMap, productType) {
+ const title = item.title ? item.title.rendered : '';
+ const link = item.link || '';
+ const slug = item.slug || '';
+
+ // Collection from taxonomy
+ const collectionIds = item.collection || [];
+ let collection = null;
+ if (collectionIds.length > 0 && collectionMap[collectionIds[0]]) {
+ collection = collectionMap[collectionIds[0]];
+ }
+
+ // Color from model_color taxonomy
+ const colorIds = item.model_color || [];
+ let colorName = null;
+ if (colorIds.length > 0 && colorMap[colorIds[0]]) {
+ colorName = colorMap[colorIds[0]];
+ }
+ // Fallback: extract from class_list
+ if (!colorName && item.class_list) {
+ const classList = Object.values(item.class_list);
+ for (const cls of classList) {
+ if (typeof cls === 'string' && cls.startsWith('model_color-')) {
+ colorName = cls.replace('model_color-', '').replace(/-/g, ' ').replace(/\b\w/g, c => c.toUpperCase());
+ }
+ }
+ }
+
+ // Image from yoast og_image
+ let imageUrl = null;
+ if (item.yoast_head_json && item.yoast_head_json.og_image) {
+ const ogImages = item.yoast_head_json.og_image;
+ if (ogImages.length > 0) imageUrl = ogImages[0].url;
+ }
+
+ // SKU: use the WP post ID prefixed with COORD- since the API doesn't expose the Ref code
+ // The slug is unique and more descriptive than the post ID
+ const mfrSku = `COORD-${item.id}`;
+
+ // In-stock status from class_list
+ let inStock = true;
+ if (item.class_list) {
+ const classList = Object.values(item.class_list);
+ if (classList.some(c => typeof c === 'string' && c.includes('outofstock'))) {
+ inStock = false;
+ }
+ }
+
+ return {
+ mfr_sku: mfrSku,
+ pattern_name: title,
+ color_name: colorName,
+ collection: collection,
+ product_type: productType,
+ image_url: imageUrl,
+ product_url: link,
+ in_stock: inStock,
+ };
+}
+
+async function main() {
+ console.log('='.repeat(70));
+ console.log(' COORDONNE WALLPAPER SCRAPER - Carmen');
+ console.log('='.repeat(70));
+ console.log(` Source: WordPress REST API (no page scraping needed)`);
+ console.log(` Table: ${TABLE}`);
+ console.log('='.repeat(70));
+
+ const startTime = Date.now();
+ await ensureTable();
+ const beforeCount = await getCatalogCount(TABLE);
+ console.log(` Current products in ${TABLE}: ${beforeCount}\n`);
+
+ // Phase 1: Fetch taxonomy maps
+ console.log('[1] Fetching taxonomy maps...');
+ const collectionMap = await fetchTaxonomyMap('collection');
+ console.log(` Collections: ${Object.keys(collectionMap).length}`);
+ const colorMap = await fetchTaxonomyMap('model_color');
+ console.log(` Colors: ${Object.keys(colorMap).length}`);
+
+ // Phase 2: Fetch all products via API
+ console.log('\n[2] Fetching products from WordPress API...');
+ let allProducts = [];
+
+ for (const cat of WALLPAPER_CATS) {
+ console.log(`\n Category: ${cat.name} (ID: ${cat.id})`);
+ let page = 1;
+ let catCount = 0;
+
+ while (page <= 50) { // safety limit
+ try {
+ const items = await fetchJSON(
+ `${BASE_URL}/wp-json/wp/v2/product?product_cat=${cat.id}&per_page=${PER_PAGE}&page=${page}`
+ );
+
+ // If we get an error object instead of array, we're past the last page
+ if (!Array.isArray(items)) {
+ console.log(` Page ${page}: end of results`);
+ break;
+ }
+ if (items.length === 0) break;
+
+ for (const item of items) {
+ allProducts.push(extractProductData(item, collectionMap, colorMap, cat.type));
+ }
+
+ catCount += items.length;
+ console.log(` Page ${page}: ${items.length} products (category total: ${catCount})`);
+
+ if (items.length < PER_PAGE) break;
+ page++;
+ await wait(500, 300); // Be respectful to the server
+ } catch (err) {
+ if (err.message && err.message.includes('rest_post_invalid_page_number')) {
+ console.log(` Page ${page}: end of results`);
+ break;
+ }
+ console.error(` Error on page ${page}: ${err.message}`);
+ // Try once more after a longer wait
+ await wait(3000, 1000);
+ page++;
+ }
+ }
+
+ console.log(` ${cat.name} total: ${catCount} products`);
+ }
+
+ console.log(`\n Total products fetched: ${allProducts.length}`);
+
+ // Dedup by mfr_sku (some products may be in both categories)
+ const seen = new Set();
+ const uniqueProducts = [];
+ for (const p of allProducts) {
+ if (!seen.has(p.mfr_sku)) {
+ seen.add(p.mfr_sku);
+ uniqueProducts.push(p);
+ }
+ }
+ console.log(` Unique products after dedup: ${uniqueProducts.length}`);
+
+ // Show samples
+ console.log('\n Sample products:');
+ for (const p of uniqueProducts.slice(0, 5)) {
+ console.log(` ${p.mfr_sku.padEnd(15)} | ${(p.pattern_name || '').substring(0, 35).padEnd(35)} | ${(p.collection || 'N/A').substring(0, 20).padEnd(20)} | ${(p.color_name || 'N/A').substring(0, 15)} | img: ${p.image_url ? 'YES' : 'no'}`);
+ }
+
+ // Phase 3: Upsert to database
+ console.log(`\n[3] Upserting ${uniqueProducts.length} products to ${TABLE}...`);
+
+ for (let i = 0; i < uniqueProducts.length; i++) {
+ const p = uniqueProducts[i];
+ try {
+ const id = await upsertProduct(TABLE, p);
+ if (id) totalInserted++;
+ else totalSkipped++;
+ } catch (err) {
+ totalErrors++;
+ if (totalErrors <= 5) console.error(` DB Error for ${p.mfr_sku}: ${err.message}`);
+ }
+ if ((i + 1) % 500 === 0) console.log(` Progress: ${i + 1}/${uniqueProducts.length} (inserted: ${totalInserted})`);
+ }
+
+ const afterCount = await getCatalogCount(TABLE);
+ const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+
+ console.log('\n' + '='.repeat(70));
+ console.log(' COORDONNE SCRAPE COMPLETE');
+ console.log('='.repeat(70));
+ console.log(` API products: ${allProducts.length}`);
+ console.log(` Unique (deduped): ${uniqueProducts.length}`);
+ console.log(` Inserted/updated: ${totalInserted}`);
+ console.log(` Skipped: ${totalSkipped}`);
+ console.log(` Errors: ${totalErrors}`);
+ console.log(` Before: ${beforeCount}`);
+ console.log(` After: ${afterCount}`);
+ console.log(` Net new: ${afterCount - beforeCount}`);
+ console.log(` Duration: ${elapsed}s`);
+ console.log('='.repeat(70));
+
+ await closePool();
+}
+
+main().catch(err => {
+ console.error('FATAL:', err);
+ closePool().then(() => process.exit(1));
+});
diff --git a/scrape-hollywood.js b/scrape-hollywood.js
new file mode 100644
index 0000000..50d2b60
--- /dev/null
+++ b/scrape-hollywood.js
@@ -0,0 +1,346 @@
+#!/usr/bin/env node
+// ============================================================================
+// Hollywood Wallcoverings Scraper
+// ============================================================================
+// Strategy: Shopify JSON API from designerwallcoverings.com
+// - hollywoodwallcoverings.com redirects to designerwallcoverings.com
+// - Single collection: hollywood-wallcovering-collection
+// - ~5000+ products, paginate products.json?limit=250&page=N
+// - Extract: SKU, pattern, color, collection, material, price, image, URL
+//
+// Target table: hollywood_catalog
+// ============================================================================
+
+const { pool, upsertProduct, wait, closePool } = require('./scraper-utils');
+const https = require('https');
+
+const TABLE = 'hollywood_catalog';
+const SHOP_BASE = 'https://www.designerwallcoverings.com';
+const COLLECTION_SLUG = 'hollywood-wallcovering-collection';
+
+let totalInserted = 0;
+let totalErrors = 0;
+let totalSkipped = 0;
+
+// ============================================================================
+// HTTP helper
+// ============================================================================
+
+const httpsAgent = new https.Agent({ rejectUnauthorized: false });
+
+function httpGetJson(url) {
+ return new Promise((resolve, reject) => {
+ const urlObj = new URL(url);
+ const options = {
+ hostname: urlObj.hostname,
+ path: urlObj.pathname + urlObj.search,
+ method: 'GET',
+ agent: httpsAgent,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+ 'Accept': 'application/json',
+ },
+ timeout: 30000,
+ };
+
+ const req = https.request(options, (res) => {
+ if (res.statusCode === 301 || res.statusCode === 302) {
+ return httpGetJson(res.headers.location).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', chunk => data += chunk);
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(data));
+ } catch (e) {
+ reject(new Error(`JSON parse error at ${url}: ${e.message}`));
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ req.end();
+ });
+}
+
+// ============================================================================
+// Parse a Shopify product into catalog format
+// ============================================================================
+
+function parseShopifyProduct(product) {
+ const title = product.title || '';
+ const tags = product.tags || [];
+ const handle = product.handle || '';
+
+ // Title format: "Abbingdon Type II Vinyl Wallpaper | Hollywood Wallcoverings"
+ // or: "Pattern Name - Color | Hollywood Wallcoverings"
+ let pattern_name = null;
+ let color_name = null;
+
+ // Remove vendor suffix
+ let cleanTitle = title.replace(/\s*\|\s*Hollywood\s+Wallcoverings\s*/i, '').trim();
+
+ // Try "Pattern - Color" format
+ const dashMatch = cleanTitle.match(/^(.+?)\s*-\s*(.+?)(?:\s+(?:Type\s+II|Vinyl|Wallpaper|Wallcovering))?$/i);
+ if (dashMatch) {
+ pattern_name = dashMatch[1].trim();
+ color_name = dashMatch[2].trim();
+ } else {
+ // Use full title as pattern name, clean up type suffixes
+ pattern_name = cleanTitle
+ .replace(/\s+Type\s+II\s+Vinyl\s+Wallpaper/i, '')
+ .replace(/\s+Type\s+II\s+Vinyl\s+Wallcovering/i, '')
+ .replace(/\s+Wallpaper/i, '')
+ .replace(/\s+Wallcovering/i, '')
+ .trim();
+ }
+
+ // Try to extract color from tags
+ if (!color_name) {
+ const colorTags = tags.filter(t =>
+ !t.match(/Hollywood|Wallcovering|Wallpaper|vinyl|display_variant|Type\s*II/i)
+ && !t.match(/^\d/)
+ && t.length > 2
+ && t.length < 30
+ );
+ // Use the tag matching the pattern name to find the color
+ const patternTag = tags.find(t => t === pattern_name);
+ if (patternTag) {
+ // Pattern tag found, color is the next non-matching descriptive tag
+ const colorCandidates = colorTags.filter(t => t !== pattern_name);
+ if (colorCandidates.length > 0) color_name = colorCandidates[0];
+ }
+ }
+
+ // Variants - get SKU and price
+ const variants = product.variants || [];
+ let mfr_sku = null;
+ let price_retail = null;
+
+ // Prefer non-sample variant
+ const fullVariant = variants.find(v => !v.sku?.toLowerCase().includes('sample'));
+ const sampleVariant = variants.find(v => v.sku?.toLowerCase().includes('sample'));
+
+ if (fullVariant) {
+ mfr_sku = fullVariant.sku;
+ price_retail = parseFloat(fullVariant.price) || null;
+ } else if (sampleVariant) {
+ mfr_sku = sampleVariant.sku.replace(/-[Ss]ample$/i, '');
+ price_retail = null;
+ }
+
+ if (!mfr_sku) mfr_sku = handle;
+
+ // Extract material from tags
+ let material = null;
+ if (tags.some(t => t.match(/vinyl/i))) material = 'Vinyl';
+ else if (tags.some(t => t.match(/100%\s*vinyl/i))) material = '100% Vinyl';
+ else if (tags.some(t => t.match(/Type\s*II/i))) material = 'Type II Vinyl';
+
+ // Product type
+ const product_type = product.product_type || 'Commercial Wallcoverings';
+
+ // Collection detection from tags
+ let collection = 'Hollywood Wallcoverings';
+ // Some tags might indicate sub-collections
+ const collTag = tags.find(t => t.match(/collection/i) && !t.match(/Hollywood/i));
+ if (collTag) collection = collTag;
+
+ // Images — capture the FULL array, not just the hero.
+ // The Shopify products.json feed exposes product.images[] (often 8+ per SKU);
+ // the old code kept only images[0] and discarded the rest. We now persist all
+ // full-resolution srcs in all_images (JSON array string) and keep image_url as
+ // the first/hero for backward compatibility.
+ const imageSrcs = (product.images || [])
+ .map(i => i && i.src)
+ .filter(Boolean);
+ const image_url = imageSrcs[0] || null;
+ const all_images = imageSrcs.length ? JSON.stringify(imageSrcs) : null;
+
+ // Product URL
+ const product_url = `${SHOP_BASE}/products/${handle}`;
+
+ // ALL DATA — capture additional feed fields the old parser dropped, into a
+ // specs jsonb blob so nothing useful from the source is thrown away.
+ const allVariantSkus = variants
+ .map(v => v && v.sku)
+ .filter(Boolean);
+ const specs = {
+ shopify_product_id: product.id ? String(product.id) : null,
+ handle,
+ title,
+ body_html: product.body_html || null, // spec/description text
+ vendor: product.vendor || null,
+ product_type: product.product_type || null,
+ published_at: product.published_at || null,
+ created_at: product.created_at || null,
+ updated_at: product.updated_at || null,
+ tags,
+ options: product.options || null, // e.g. Title / sample-vs-roll
+ variant_skus: allVariantSkus, // all variant SKUs (roll + sample)
+ image_count: imageSrcs.length,
+ image_srcs: imageSrcs, // redundant w/ all_images, kept in specs for completeness
+ };
+
+ return {
+ mfr_sku,
+ pattern_name,
+ color_name,
+ collection,
+ product_type,
+ material,
+ price_retail,
+ image_url,
+ all_images,
+ product_url,
+ shopify_product_id: product.id ? String(product.id) : null,
+ specs: JSON.stringify(specs),
+ };
+}
+
+// ============================================================================
+// MAIN
+// ============================================================================
+
+async function main() {
+ console.log('='.repeat(70));
+ console.log(' HOLLYWOOD WALLCOVERINGS SCRAPER');
+ console.log('='.repeat(70));
+ console.log(` Source: Shopify JSON API (${SHOP_BASE})`);
+ console.log(` Collection: ${COLLECTION_SLUG}`);
+ console.log(` Target table: ${TABLE}`);
+ console.log('='.repeat(70));
+
+ const startTime = Date.now();
+
+ // Ensure table exists
+ try {
+ await pool.query(`SELECT 1 FROM ${TABLE} LIMIT 1`);
+ console.log(`\n Table "${TABLE}" exists.`);
+ } catch (err) {
+ console.log(`\n Table "${TABLE}" does not exist, creating...`);
+ await pool.query(`
+ CREATE TABLE IF NOT EXISTS ${TABLE} (
+ id SERIAL PRIMARY KEY,
+ mfr_sku TEXT UNIQUE,
+ dw_sku TEXT,
+ pattern_name TEXT,
+ color_name TEXT,
+ collection TEXT,
+ product_type TEXT,
+ width TEXT,
+ length TEXT,
+ repeat_v TEXT,
+ repeat_h TEXT,
+ material TEXT,
+ price_retail NUMERIC(10,2),
+ price_trade NUMERIC(10,2),
+ image_url TEXT,
+ product_url TEXT,
+ in_stock BOOLEAN DEFAULT TRUE,
+ discontinued BOOLEAN DEFAULT FALSE,
+ shopify_product_id TEXT,
+ last_scraped TIMESTAMPTZ,
+ created_at TIMESTAMPTZ DEFAULT NOW(),
+ updated_at TIMESTAMPTZ DEFAULT NOW()
+ );
+ `);
+ console.log(` Table created.`);
+ }
+
+ // Ensure the columns we now write exist (idempotent — safe on every run).
+ // all_images stores the full image array (JSON), specs stores the rest of the
+ // feed (body_html, all variant SKUs, options, etc.) so no source data is lost.
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS all_images TEXT`);
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS shopify_product_id TEXT`);
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS specs JSONB`);
+
+ // Paginate through collection
+ let page = 1;
+ let grandTotal = 0;
+ const maxPages = 60; // Safety limit (collection ~3,300 products / 250 = ~14 pages)
+
+ while (page <= maxPages) {
+ try {
+ const url = `${SHOP_BASE}/collections/${COLLECTION_SLUG}/products.json?limit=250&page=${page}`;
+ const data = await httpGetJson(url);
+ const products = data.products || [];
+
+ if (products.length === 0) break;
+
+ for (const product of products) {
+ const parsed = parseShopifyProduct(product);
+ if (!parsed || !parsed.mfr_sku) {
+ totalSkipped++;
+ continue;
+ }
+
+ try {
+ const id = await upsertProduct(TABLE, parsed);
+ if (id) totalInserted++;
+ else totalSkipped++;
+ } catch (err) {
+ totalErrors++;
+ }
+ grandTotal++;
+ }
+
+ console.log(` Page ${page}: ${products.length} products (total: ${grandTotal}, inserted: ${totalInserted})`);
+
+ if (products.length < 250) break;
+ page++;
+ await wait(500, 300);
+ } catch (err) {
+ console.error(` Error on page ${page}: ${err.message}`);
+ totalErrors++;
+ // Retry once after delay
+ await wait(3000, 1000);
+ try {
+ const url = `${SHOP_BASE}/collections/${COLLECTION_SLUG}/products.json?limit=250&page=${page}`;
+ const data = await httpGetJson(url);
+ const products = data.products || [];
+ if (products.length === 0) break;
+ for (const product of products) {
+ const parsed = parseShopifyProduct(product);
+ if (parsed && parsed.mfr_sku) {
+ const id = await upsertProduct(TABLE, parsed);
+ if (id) totalInserted++;
+ grandTotal++;
+ }
+ }
+ console.log(` Page ${page} (retry): ${products.length} products`);
+ } catch (retryErr) {
+ console.error(` Retry failed for page ${page}, skipping.`);
+ }
+ page++;
+ await wait(1000, 500);
+ }
+ }
+
+ // Final stats
+ const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+ let count = 0;
+ try {
+ const res = await pool.query(`SELECT COUNT(*) as cnt FROM ${TABLE}`);
+ count = parseInt(res.rows[0].cnt);
+ } catch (_) {}
+
+ console.log('\n' + '='.repeat(70));
+ console.log(' SCRAPE COMPLETE - HOLLYWOOD WALLCOVERINGS');
+ console.log('='.repeat(70));
+ console.log(` Products processed: ${grandTotal}`);
+ console.log(` Products inserted/updated: ${totalInserted}`);
+ console.log(` Skipped: ${totalSkipped}`);
+ console.log(` Errors: ${totalErrors}`);
+ console.log(` Total in ${TABLE}: ${count}`);
+ console.log(` Duration: ${elapsed}s`);
+ console.log('='.repeat(70));
+
+ await closePool();
+}
+
+main().catch(err => {
+ console.error('FATAL:', err);
+ closePool().then(() => process.exit(1));
+});
(oldest)
·
back to Dw Vendor Scrapers Local
·
Add recrawl-designers-guild.js: additive all_images re-crawl ec029c3 →