← back to Dw Cowtan Recrawl
scaffold: cowtan/osborne/graham_brown recrawl drivers + probes
c8b0219c14c9ab48ae339bfb905f512ee41c7536 · 2026-06-10 11:24:02 -0700 · SteveStudio2
Files touched
A .gitignoreA probe.jsA probe2.jsA probe3.jsA probe4.jsA probe5.jsA recrawl-cowtan.jsA reference/recrawl-graham_brown.jsA reference/recrawl-osborne.jsA reference/scrape-cowtan.jsA reference/scrape-hollywood.js
Diff
commit c8b0219c14c9ab48ae339bfb905f512ee41c7536
Author: SteveStudio2 <steve@designerwallcoverings.com>
Date: Wed Jun 10 11:24:02 2026 -0700
scaffold: cowtan/osborne/graham_brown recrawl drivers + probes
---
.gitignore | 8 +
probe.js | 34 ++++
probe2.js | 38 +++++
probe3.js | 23 +++
probe4.js | 31 ++++
probe5.js | 37 ++++
recrawl-cowtan.js | 158 +++++++++++++++++
reference/recrawl-graham_brown.js | 195 +++++++++++++++++++++
reference/recrawl-osborne.js | 77 +++++++++
reference/scrape-cowtan.js | 71 ++++++++
reference/scrape-hollywood.js | 346 ++++++++++++++++++++++++++++++++++++++
11 files changed, 1018 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..1924158
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,8 @@
+node_modules/
+.env*
+tmp/
+*.log
+.DS_Store
+dist/
+build/
+.next/
diff --git a/probe.js b/probe.js
new file mode 100644
index 0000000..673ffd8
--- /dev/null
+++ b/probe.js
@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+const https = require('https');
+function get(u, m = 4) {
+ return new Promise((res, rej) => {
+ https.get(u, { timeout: 25000, headers: { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) AppleWebKit/537.36 Chrome/120 Safari/537.36', 'Accept': 'text/html' } }, r => {
+ if ((r.statusCode == 301 || r.statusCode == 302) && r.headers.location && m > 0) {
+ r.resume();
+ const loc = r.headers.location.startsWith('http') ? r.headers.location : 'https://designs.cowtan.com' + r.headers.location;
+ return get(loc, m - 1).then(res).catch(rej);
+ }
+ if (r.statusCode != 200) { r.resume(); return rej(new Error('HTTP ' + r.statusCode)); }
+ let d = ''; r.on('data', c => d += c); r.on('end', () => res(d));
+ }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); });
+ });
+}
+(async () => {
+ const urls = ['https://designs.cowtan.com/Design/1020-01', 'https://designs.cowtan.com/Design/20527-01'];
+ for (const url of urls) {
+ console.log('\n==== ' + url + ' ====');
+ try {
+ const html = await get(url);
+ console.log('HTML len:', html.length);
+ const imgs = [...html.matchAll(/(https:\/\/d2mq91o692rj7w\.cloudfront\.net\/[^"'\s)]+)/g)].map(m => m[1]);
+ const uniq = [...new Set(imgs)];
+ console.log('distinct cloudfront img urls:', uniq.length);
+ uniq.slice(0, 30).forEach(u => console.log(' ', u));
+ console.log('has .colour-options:', /colour-options/.test(html));
+ const co = [...new Set([...html.matchAll(/data-stockcode="([^"]+)"/g)].map(m => m[1]))];
+ console.log('colorway stockcodes:', co.length, co.slice(0, 12).join(', '));
+ const cm = html.match(/Color:\s*([a-zA-Z][a-zA-Z\s,&]{1,30})/i);
+ console.log('Color: match ->', cm ? cm[1].trim() : '(none)');
+ } catch (e) { console.error('ERR', e.message); }
+ }
+})();
diff --git a/probe2.js b/probe2.js
new file mode 100644
index 0000000..a9177da
--- /dev/null
+++ b/probe2.js
@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+// Probe the cowtan /Search AJAX API raw product object + hunt for a design-detail JSON endpoint.
+const https = require('https');
+function httpPost(url, body) {
+ return new Promise((resolve, reject) => {
+ const p = new URL(url);
+ const req = https.request({ hostname: p.hostname, port: 443, path: p.pathname, method: 'POST', timeout: 30000,
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(body), 'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json', 'X-Requested-With': 'XMLHttpRequest' } },
+ res => { if (res.statusCode == 301 || res.statusCode == 302) { return httpPost(res.headers.location, body).then(resolve).catch(reject); } let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(new Error('JSON ' + e.message + ' len=' + d.length)); } }); });
+ req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ req.write(body); req.end();
+ });
+}
+function httpGet(url, m = 4) {
+ return new Promise((res, rej) => { https.get(url, { timeout: 25000, headers: { 'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json, text/html' } }, r => {
+ if ((r.statusCode == 301 || r.statusCode == 302) && r.headers.location && m > 0) { r.resume(); const loc = r.headers.location.startsWith('http') ? r.headers.location : 'https://designs.cowtan.com' + r.headers.location; return httpGet(loc, m - 1).then(res).catch(rej); }
+ let d = ''; r.on('data', c => d += c); r.on('end', () => res({ status: r.statusCode, body: d, ct: r.headers['content-type'] })); }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); }); });
+}
+function buildBody(pi, tv) { const p = ['Keywords=', 'NewOnly=false']; ['F', 'W', 'T'].forEach((t, i) => { p.push('Types%5B' + i + '%5D.Value=' + t); p.push('Types%5B' + i + '%5D.Selected=' + (t === tv ? 'true' : 'false')); }); p.push('PageIndex=' + pi); return p.join('&'); }
+(async () => {
+ console.log('=== /Search raw product[0] keys ===');
+ const f = await httpPost('https://designs.cowtan.com/Search', buildBody(0, 'W'));
+ const prod = (f.Products && f.Products[0] && f.Products[0][0]) || null;
+ if (prod) { console.log('keys:', Object.keys(prod).join(', ')); console.log(JSON.stringify(prod, null, 1).slice(0, 1500)); }
+ else console.log('no product, top keys:', Object.keys(f).join(', '));
+ // hunt detail endpoints
+ const code = prod ? prod.StockCode : '1020-01';
+ const cands = [
+ '/Design/Detail/' + code, '/Design/Get/' + code, '/api/design/' + code,
+ '/Product/' + code, '/Design/Product?stockCode=' + encodeURIComponent(code),
+ '/Search/Product?stockCode=' + encodeURIComponent(code), '/Design/Data/' + code
+ ];
+ console.log('\n=== detail endpoint probes for', code, '===');
+ for (const c of cands) {
+ try { const r = await httpGet('https://designs.cowtan.com' + c); console.log(c, '->', r.status, (r.ct || '').split(';')[0], 'len=' + r.body.length, /flatshots|RoomScene|Images/.test(r.body) ? 'HAS-IMG-HINT' : ''); }
+ catch (e) { console.log(c, '-> ERR', e.message); }
+ }
+})().catch(e => console.error('FATAL', e.message));
diff --git a/probe3.js b/probe3.js
new file mode 100644
index 0000000..bc451d1
--- /dev/null
+++ b/probe3.js
@@ -0,0 +1,23 @@
+#!/usr/bin/env node
+const https = require('https');
+function raw(method, url, body) {
+ return new Promise((resolve, reject) => {
+ const p = new URL(url);
+ const opts = { hostname: p.hostname, port: 443, path: p.pathname + p.search, method, timeout: 30000,
+ headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120 Safari/537.36', 'Accept': 'application/json, text/javascript, */*; q=0.01', 'X-Requested-With': 'XMLHttpRequest' } };
+ if (body) { opts.headers['Content-Type'] = 'application/x-www-form-urlencoded'; opts.headers['Content-Length'] = Buffer.byteLength(body); }
+ const req = https.request(opts, res => { let d = ''; res.on('data', c => d += c); res.on('end', () => resolve({ status: res.statusCode, loc: res.headers.location, ct: res.headers['content-type'], body: d })); });
+ req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ if (body) req.write(body); req.end();
+ });
+}
+function buildBody(pi, tv) { const p = ['Keywords=', 'NewOnly=false']; ['F', 'W', 'T'].forEach((t, i) => { p.push('Types%5B' + i + '%5D.Value=' + t); p.push('Types%5B' + i + '%5D.Selected=' + (t === tv ? 'true' : 'false')); }); p.push('PageIndex=' + pi); return p.join('&'); }
+(async () => {
+ const r = await raw('POST', 'https://designs.cowtan.com/Search', buildBody(0, 'W'));
+ console.log('POST /Search ->', r.status, r.ct, 'loc=' + (r.loc || '-'), 'len=' + r.body.length);
+ console.log('--- first 600 chars ---\n' + r.body.slice(0, 600));
+ // try the apex / www host
+ for (const h of ['https://designs.cowtan.com/', 'https://www.cowtan.com/', 'https://www.cowtantout.com/']) {
+ try { const x = await raw('GET', h); console.log('\nGET', h, '->', x.status, (x.ct || '').split(';')[0], 'len=' + x.body.length, 'loc=' + (x.loc || '-')); } catch (e) { console.log('GET', h, 'ERR', e.message); }
+ }
+})().catch(e => console.error('FATAL', e.message));
diff --git a/probe4.js b/probe4.js
new file mode 100644
index 0000000..1c8a670
--- /dev/null
+++ b/probe4.js
@@ -0,0 +1,31 @@
+#!/usr/bin/env node
+// Sniff the cowtan SPA shell + JS bundles for the new API endpoints (feed-first discovery).
+const https = require('https');
+function get(u, m = 5) {
+ return new Promise((res, rej) => { https.get(u, { timeout: 25000, headers: { 'User-Agent': 'Mozilla/5.0', 'Accept': '*/*' } }, r => {
+ if ((r.statusCode == 301 || r.statusCode == 302) && r.headers.location && m > 0) { r.resume(); const loc = r.headers.location.startsWith('http') ? r.headers.location : 'https://designs.cowtan.com' + r.headers.location; return get(loc, m - 1).then(res).catch(rej); }
+ let d = ''; r.on('data', c => d += c); r.on('end', () => res({ status: r.statusCode, body: d })); }).on('error', rej).on('timeout', function () { this.destroy(); rej(new Error('timeout')); }); });
+}
+(async () => {
+ const shell = await get('https://designs.cowtan.com/');
+ console.log('shell', shell.status, 'len', shell.body.length);
+ // script srcs
+ const scripts = [...new Set([...shell.body.matchAll(/<script[^>]+src="([^"]+)"/g)].map(m => m[1]))];
+ console.log('scripts:', scripts.length); scripts.forEach(s => console.log(' ', s));
+ // inline api hints in shell
+ const apiHints = [...new Set([...shell.body.matchAll(/["'`](\/(?:api|Search|Design|Product|Catalog)[^"'`\s]*)["'`]/gi)].map(m => m[1]))];
+ console.log('shell api-path hints:', apiHints.slice(0, 30).join(' '));
+ // fetch each same-origin/relative JS and grep for fetch/axios/api paths
+ for (const s of scripts) {
+ const url = s.startsWith('http') ? s : (s.startsWith('//') ? 'https:' + s : 'https://designs.cowtan.com' + s);
+ if (!/cowtan/.test(url)) { console.log('skip 3rd-party', url); continue; }
+ try {
+ const js = await get(url);
+ const paths = [...new Set([...js.body.matchAll(/["'`](\/(?:api|Search|Design|Product|Catalog|Get|List|Data)[A-Za-z0-9_\/\-?=&.]*)["'`]/g)].map(m => m[1]))];
+ const fetches = [...new Set([...js.body.matchAll(/(?:fetch|axios\.\w+|\.get|\.post)\(\s*["'`]([^"'`]+)["'`]/g)].map(m => m[1]))];
+ console.log('\nJS', url, 'len', js.body.length);
+ console.log(' api-ish paths:', paths.slice(0, 40).join(' ') || '(none)');
+ console.log(' fetch/axios urls:', fetches.slice(0, 40).join(' ') || '(none)');
+ } catch (e) { console.log('JS', url, 'ERR', e.message); }
+ }
+})().catch(e => console.error('FATAL', e.message));
diff --git a/probe5.js b/probe5.js
new file mode 100644
index 0000000..b224117
--- /dev/null
+++ b/probe5.js
@@ -0,0 +1,37 @@
+#!/usr/bin/env node
+// Verify the NEW cowtan API: /api/Product/Detail/<code> and /api/ProductSearch
+const https = require('https');
+function req(method, path, body) {
+ return new Promise((resolve, reject) => {
+ const opts = { hostname: 'designs.cowtan.com', port: 443, path, method, timeout: 30000,
+ headers: { 'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json, */*', 'X-Requested-With': 'XMLHttpRequest' } };
+ if (body) { opts.headers['Content-Type'] = 'application/json'; opts.headers['Content-Length'] = Buffer.byteLength(body); }
+ const r = https.request(opts, res => { let d = ''; res.on('data', c => d += c); res.on('end', () => resolve({ status: res.statusCode, ct: res.headers['content-type'], body: d })); });
+ r.on('error', reject); r.on('timeout', () => { r.destroy(); reject(new Error('timeout')); });
+ if (body) r.write(body); r.end();
+ });
+}
+function imgUrls(obj) {
+ const out = new Set(); JSON.stringify(obj).replace(/https?:\/\/[^"'\\\s]+\.(?:jpg|jpeg|png|webp)/gi, m => { out.add(m); return m; });
+ return [...out];
+}
+(async () => {
+ for (const code of ['1020-01', '20527-01']) {
+ const r = await req('GET', '/api/Product/Detail/' + encodeURIComponent(code));
+ console.log('\n=== GET /api/Product/Detail/' + code + ' ->', r.status, (r.ct || '').split(';')[0], 'len=' + r.body.length);
+ if (r.status == 200 && /json/.test(r.ct || '')) {
+ let j; try { j = JSON.parse(r.body); } catch (e) { console.log(' parse err', e.message); continue; }
+ console.log(' top keys:', Object.keys(j).join(', '));
+ const imgs = imgUrls(j);
+ console.log(' image urls found:', imgs.length);
+ imgs.slice(0, 20).forEach(u => console.log(' ', u));
+ } else { console.log(' body head:', r.body.slice(0, 200).replace(/\n/g, ' ')); }
+ }
+ // ProductSearch POST
+ for (const body of [JSON.stringify({ Keywords: '', PageIndex: 0 }), 'Keywords=&PageIndex=0']) {
+ const isJson = body.startsWith('{');
+ const r = await req('POST', '/api/ProductSearch', body);
+ console.log('\n=== POST /api/ProductSearch (' + (isJson ? 'json' : 'form') + ') ->', r.status, (r.ct || '').split(';')[0], 'len=' + r.body.length);
+ console.log(' head:', r.body.slice(0, 240).replace(/\n/g, ' '));
+ }
+})().catch(e => console.error('FATAL', e.message));
diff --git a/recrawl-cowtan.js b/recrawl-cowtan.js
new file mode 100644
index 0000000..c59055e
--- /dev/null
+++ b/recrawl-cowtan.js
@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+// recrawl-cowtan.js — additive all_images + full-spec backfill for cowtan_tout.
+//
+// WHY THIS EXISTS: designs.cowtan.com re-platformed to an Alpine.js SPA after the
+// Apr-30 crawl. The old `POST /Search` AJAX endpoint that scrape-cowtan.js depends on
+// now returns HTTP 404, so the deployed scraper is broken AND it never captured
+// all_images in the first place (it built a single image_url from StockCode).
+//
+// This driver uses the NEW, verified feed-first API:
+// GET /api/Product/Detail/<StockCode> -> JSON array of sibling colourways, each with
+// full specs (Width, RepeatV/H, Composition, Qualities, Price, InStock, Type) and
+// RoomShotPictureIDs (room-scene image IDs).
+// Hi-res flatshot per colourway is built from the SPA's own template:
+// https://d2mq91o692rj7w.cloudfront.net/flatshots/2017-main/<urlSafeStockCode>_m.jpg
+// (thumb = 2017-thumb/<code>_t.jpg). urlSafeStockCode = StockCode.replaceAll('/','-').
+//
+// all_images per row = JSON array of [ main flatshot, ...room-shot urls ] for THAT colourway.
+// Each sibling colourway is its own mfr_sku row and gets its own all_images.
+//
+// ADDITIVE & IDEMPOTENT: only UPDATEs all_images / specs columns on existing rows
+// (matched by mfr_sku). Never deletes, never inserts net-new SKUs, never archives.
+// One Detail fetch covers a whole pattern (all its colourways), so we fetch by distinct
+// ProductCode to minimise requests (~one call per pattern, not per colourway).
+//
+// USAGE (run ON Kamatera prod, gated — see pending-approval/cowtan_tout-recrawl.md):
+// node recrawl-cowtan.js --net-new --all-images # backfill rows missing all_images
+// node recrawl-cowtan.js --net-new --all-images --limit 50 --dry-run # safe probe
+//
+// ROOM-SHOT URL PATTERN: RoomShotPictureIDs was empty on the probed samples. Before the
+// full prod run, confirm the room-shot CDN path on a SKU that HAS room shots and set
+// ROOMSHOT_URL() below. Until confirmed, room shots are skipped (main flatshot still
+// populates all_images, clearing the gate). This is the one item to verify when attended.
+
+const https = require('https');
+
+// ---- config -------------------------------------------------------------
+const TABLE = process.env.COWTAN_TABLE || 'vendor_catalog'; // local: vendor_catalog (vendor_code filter); prod fleet table may be cowtan_tout_catalog
+const VENDOR_CODE = 'cowtan_tout';
+const CDN = 'https://d2mq91o692rj7w.cloudfront.net/flatshots/';
+const API = 'https://designs.cowtan.com';
+const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const CONCURRENCY = 3; // polite — vendor is the same one we re-platformed off of
+const SLEEP_MS = 350;
+
+const argv = process.argv.slice(2);
+const DRY = argv.includes('--dry-run');
+const LIMIT = (() => { const i = argv.indexOf('--limit'); return i > -1 ? parseInt(argv[i + 1], 10) : 0; })();
+
+// pg pool: prefer the fleet scraper-utils pool if present (prod), else local DATABASE_URL.
+let pool;
+try {
+ ({ pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils'));
+} catch (_) {
+ const { Pool } = require('pg');
+ pool = new Pool({ connectionString: process.env.DATABASE_URL || 'postgres://stevestudio2@localhost/dw_unified' });
+}
+
+// ---- helpers ------------------------------------------------------------
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 200)));
+const urlSafe = (code) => String(code).replaceAll('/', '-');
+const mainFlatshot = (code) => CDN + '2017-main/' + urlSafe(code) + '_m.jpg';
+// CONFIRM-WHEN-ATTENDED: room-shot URL builder. Returns [] until the pattern is verified.
+const ROOMSHOT_URL = (picId) => null; // e.g. CDN + 'rooms/' + picId + '.jpg' — TBD, see header note.
+
+function getJson(path, maxRedirects = 4) {
+ return new Promise((resolve, reject) => {
+ const req = https.request({ hostname: 'designs.cowtan.com', path, method: 'GET', timeout: 25000,
+ headers: { 'User-Agent': UA, 'Accept': 'application/json, */*', 'X-Requested-With': 'XMLHttpRequest' } }, res => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ res.resume();
+ const loc = res.headers.location.replace(API, '');
+ return getJson(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(new Error('JSON ' + e.message)); } });
+ });
+ req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ req.end();
+ });
+}
+
+function buildAllImages(entry) {
+ const imgs = [mainFlatshot(entry.StockCode)];
+ for (const pid of (entry.RoomShotPictureIDs || [])) { const u = ROOMSHOT_URL(pid); if (u) imgs.push(u); }
+ return [...new Set(imgs)];
+}
+
+function specJson(entry) {
+ return {
+ width: entry.Width || null, repeat_v: entry.RepeatV || null, repeat_h: entry.RepeatH || null,
+ finish: entry.Finish || null, type: entry.Type || null, weight: entry.Weight || null,
+ composition: entry.Composition || null, qualities: entry.Qualities || null,
+ testing: entry.Testing || null, price: entry.Price || null, in_stock: entry.InStock ?? null,
+ availability: entry.Availability || null, pattern_book: entry.PatternBook || null,
+ brand_name: entry.BrandName || null, room_shot_picture_ids: entry.RoomShotPictureIDs || [],
+ };
+}
+
+// ---- main ---------------------------------------------------------------
+(async () => {
+ // worklist: rows still missing all_images. We crawl by distinct ProductCode (pattern)
+ // since one Detail call returns every colourway of that pattern.
+ const where = `vendor_code = $1 AND (all_images IS NULL OR all_images = '' OR all_images = '[]' OR all_images = 'null')`;
+ const q = `SELECT DISTINCT split_part(mfr_sku,'-',1) AS product_code, min(mfr_sku) AS seed_sku
+ FROM ${TABLE} WHERE ${where} GROUP BY 1 ORDER BY 1` + (LIMIT ? ` LIMIT ${LIMIT}` : '');
+ const { rows } = await pool.query(q, [VENDOR_CODE]).catch(async (e) => {
+ // fleet per-vendor table has no vendor_code column — fall back to plain table scan.
+ if (/vendor_code/.test(e.message)) {
+ const w2 = `(all_images IS NULL OR all_images='' OR all_images='[]' OR all_images='null')`;
+ return pool.query(`SELECT DISTINCT split_part(mfr_sku,'-',1) AS product_code, min(mfr_sku) AS seed_sku FROM ${TABLE} WHERE ${w2} GROUP BY 1 ORDER BY 1` + (LIMIT ? ` LIMIT ${LIMIT}` : ''));
+ }
+ throw e;
+ });
+
+ console.log(`Re-crawling ${rows.length} distinct cowtan patterns (table=${TABLE}, dry=${DRY})...`);
+ let patterns = 0, skusUpdated = 0, errs = 0, idx = 0;
+
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const seed = rows[i].seed_sku;
+ try {
+ const detail = await getJson('/api/Product/Detail/' + encodeURIComponent(seed));
+ const entries = Array.isArray(detail) ? detail : Object.values(detail);
+ for (const e of entries) {
+ if (!e || !e.StockCode) continue;
+ const all = JSON.stringify(buildAllImages(e));
+ const specs = JSON.stringify(specJson(e));
+ if (DRY) { skusUpdated++; if (skusUpdated <= 5) console.log(' [dry]', e.StockCode, all); continue; }
+ // additive UPDATE only — never insert/delete. Match by mfr_sku within this vendor.
+ const r = await pool.query(
+ `UPDATE ${TABLE} SET all_images = $1, specs = COALESCE(specs,'{}'::jsonb) || $2::jsonb, updated_at = NOW()
+ WHERE mfr_sku = $3` + (/vendor_catalog/.test(TABLE) ? ` AND vendor_code = '${VENDOR_CODE}'` : ''),
+ [all, specs, e.StockCode]
+ ).catch(async (err) => {
+ // table may not have a jsonb specs column — degrade to all_images-only.
+ if (/column .*specs/.test(err.message)) {
+ return pool.query(`UPDATE ${TABLE} SET all_images = $1, updated_at = NOW() WHERE mfr_sku = $2` + (/vendor_catalog/.test(TABLE) ? ` AND vendor_code='${VENDOR_CODE}'` : ''), [all, e.StockCode]);
+ }
+ throw err;
+ });
+ if (r.rowCount) skusUpdated += r.rowCount;
+ }
+ patterns++;
+ if (patterns % 50 === 0) console.log(` ${patterns}/${rows.length} patterns | ${skusUpdated} skus updated | ${errs} errs`);
+ await sleep(SLEEP_MS);
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0) console.error(` err at ${seed}: ${e.message}`);
+ await sleep(800);
+ }
+ }
+ }
+
+ await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+ console.log(`\nDONE. patterns=${patterns} sku_rows_updated=${skusUpdated} errors=${errs}`);
+ await (pool.end ? pool.end() : Promise.resolve());
+})().catch(e => { console.error('FATAL', e); (pool.end ? pool.end() : Promise.resolve()).finally(() => process.exit(1)); });
diff --git a/reference/recrawl-graham_brown.js b/reference/recrawl-graham_brown.js
new file mode 100644
index 0000000..8c97283
--- /dev/null
+++ b/reference/recrawl-graham_brown.js
@@ -0,0 +1,195 @@
+#!/usr/bin/env node
+/*
+ * recrawl-graham_brown.js — additive full-page all_images re-crawl for Graham & Brown.
+ *
+ * WHY: 3117/3125 graham_brown_catalog rows carry only a primary image_url and an
+ * empty all_images, which fails Steve's "always pull all data and images"
+ * completeness gate (a row is import-ready only when the crawl captured the FULL
+ * product page incl. ALL images). This driver re-crawls each product page and
+ * fills all_images + room_setting_url WITHOUT touching anything else (additive).
+ *
+ * SOURCE: grahambrown.com is a BigCommerce Stencil store (store hash s-7psb023nit).
+ * Each PDP renders this product's gallery AND a "related products" carousel, both
+ * on the same cdn11.bigcommerce.com host — so a naive "all CDN images" grab would
+ * pollute all_images with other SKUs' thumbnails. The reliable discriminator: the
+ * THIS-product gallery images embed the product's mfr_sku in the filename
+ * (e.g. 120628_TILE_..., 120628_ROOMSET_...). We filter on that.
+ *
+ * SAFETY: additive only. We UPDATE all_images / room_setting_url ONLY on rows where
+ * all_images is currently empty. No inserts, no deletes, no Shopify writes, no
+ * column other than all_images / room_setting_url / updated_at touched.
+ *
+ * USAGE
+ * # local dry-run (no DB write needed; prints extracted images per URL):
+ * node recrawl-graham_brown.js --dry-run --limit 5
+ *
+ * # prod (Kamatera) — fills empty all_images for every missing row:
+ * DATABASE_URL=postgres://... node recrawl-graham_brown.js --all-images
+ *
+ * # prod, net-net-new only (the 240 rows not yet on Shopify) — fastest path to
+ * # making the genuinely new SKUs import-ready:
+ * DATABASE_URL=postgres://... node recrawl-graham_brown.js --net-new --all-images
+ *
+ * FLAGS
+ * --dry-run fetch + extract + print, never write DB (DATABASE_URL optional)
+ * --net-new restrict to rows with no shopify_product_id AND no shopify sku match
+ * --all-images (default behaviour) capture all_images + room images; kept explicit
+ * --limit N cap number of product pages processed
+ */
+'use strict';
+
+const https = require('https');
+// pg is required lazily inside main() so --dry-run works without the module installed.
+
+const ARGS = process.argv.slice(2);
+const DRY_RUN = ARGS.includes('--dry-run');
+const NET_NEW = ARGS.includes('--net-new');
+const LIMIT = (() => { const i = ARGS.indexOf('--limit'); return i >= 0 ? parseInt(ARGS[i + 1], 10) || 0 : 0; })();
+
+const TABLE = 'graham_brown_catalog';
+const STORE_HASH = 's-7psb023nit';
+const SITE_BASE = 'https://www.grahambrown.com';
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+const CONCURRENCY = 3;
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+function httpGet(url, maxRedirects = 4) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, Accept: 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' } }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
+ res.resume();
+ return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', (c) => (d += c)); res.on('end', () => resolve(d)); res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+
+/*
+ * Extract THIS product's gallery from a Graham & Brown BigCommerce PDP.
+ * Returns { all_images: string[], room_setting_url: string|null }.
+ * Strategy: collect every cdn11/<STORE_HASH>/.../products/.../<...>.jpg whose
+ * filename contains the product's mfr_sku, normalise each to the "original" size
+ * variant, dedupe by image path, and order TILE first (the swatch shoppers expect
+ * as primary) then the rest. room_setting_url = first ROOMSET/ROLLSHOT frame.
+ */
+function extractGrahamImages(html, mfrSku, productUrl) {
+ const sku = String(mfrSku || '').trim();
+ if (!sku) return { all_images: [], room_setting_url: null };
+ // Paint / multi-variant "master" pages live at .../<masterNum>-master/ where the
+ // URL number can differ from this row's mfr_sku. Accept either as the filename key.
+ const masterNum = (String(productUrl || '').match(/\/(\d+)-master\//) || [])[1] || '';
+ const keys = [sku, masterNum].filter(Boolean);
+ const matchesKey = (fname) => keys.some((k) => fname.includes(k));
+ const re = new RegExp(`https://cdn11\\.bigcommerce\\.com/${STORE_HASH}/images/stencil/[^"'\\\\ ]+`, 'g');
+ const seen = new Set();
+ const imgs = [];
+ let m;
+ while ((m = re.exec(html)) !== null) {
+ let url = m[0]
+ .replace(/\\\//g, '/') // un-escape JSON slashes
+ .replace(/\\/g, '') // drop any stray backslashes
+ .replace(/&/g, '&')
+ .replace(/[?].*$/, '');
+ if (!/\.(jpg|jpeg|png|webp)$/i.test(url)) continue; // skip truncated / non-image hits
+ // only THIS product's frames — filename carries the mfr_sku (or master number)
+ const fname = url.split('/').pop();
+ if (!matchesKey(fname)) continue;
+ // normalise size token (100w / 340w / 1200w / 1280w / original) -> original
+ url = url.replace(/\/stencil\/(?:\d+w|\d+x\d+|original)\//, '/stencil/original/');
+ const pathKey = url.replace(/__\d+\.\d+\.(jpg|jpeg|png|webp)$/i, ''); // strip BC cache-bust suffix for dedupe
+ if (seen.has(pathKey)) continue;
+ seen.add(pathKey);
+ imgs.push(url);
+ }
+ // order: TILE first, then ROLLSHOT/DIGITAL ROLL, then ROOMSET/FLATLAY
+ const rank = (u) => {
+ const f = u.toUpperCase();
+ if (f.includes('_TILE')) return 0;
+ if (f.includes('ROLLSHOT') || f.includes('DIGITAL')) return 1;
+ if (f.includes('FLATLAY')) return 2;
+ if (f.includes('ROOMSET') || f.includes('ROOM')) return 3;
+ return 4;
+ };
+ imgs.sort((a, b) => rank(a) - rank(b));
+ const room = imgs.find((u) => /ROOMSET|ROOM|ROLLSHOT/i.test(u)) || null;
+ return { all_images: imgs, room_setting_url: room };
+}
+
+async function main() {
+ const pool = (DATABASE_URL_PRESENT()) ? new (require('pg').Pool)({ connectionString: process.env.DATABASE_URL }) : null;
+ if (!pool && !DRY_RUN) {
+ console.error('FATAL: DATABASE_URL not set and not --dry-run. Refusing to run.');
+ process.exit(2);
+ }
+
+ // Build the worklist. Rows missing all_images that have a product_url to fetch.
+ let where = `(all_images IS NULL OR all_images = '' OR all_images = '[]') AND product_url IS NOT NULL AND product_url <> ''`;
+ if (NET_NEW) {
+ where += ` AND shopify_product_id IS NULL
+ AND NOT EXISTS (SELECT 1 FROM shopify_products s
+ WHERE s.sku = g.dw_sku OR s.sku = g.dw_sku || '-Sample' OR s.sku ILIKE g.dw_sku || '-%')`;
+ }
+ let rows;
+ if (pool) {
+ const q = `SELECT g.id, g.dw_sku, g.mfr_sku, split_part(g.product_url,'?',1) AS url
+ FROM ${TABLE} g WHERE ${where} ORDER BY g.dw_sku ${LIMIT ? 'LIMIT ' + LIMIT : ''}`;
+ ({ rows } = await pool.query(q));
+ } else {
+ // dry-run with no DB: hit a couple of known sample URLs so the extractor is exercised.
+ rows = [
+ // real net-net-new worklist rows: one wallpaper, one paint "master" (exercises the URL master-number fallback)
+ { id: 0, dw_sku: 'DWGB-200116', mfr_sku: '120628', url: 'https://www.grahambrown.com/us/product/wallflower-night-garden-wallpaper/120628-master/' },
+ { id: 0, dw_sku: 'DWGB-200555', mfr_sku: '109732', url: 'https://www.grahambrown.com/us/product/g-b-white-paint/109729-master/' },
+ ].slice(0, LIMIT || 2);
+ }
+
+ console.log(`Re-crawling ${rows.length} graham_brown product page(s)${NET_NEW ? ' (net-net-new only)' : ''}${DRY_RUN ? ' [DRY RUN]' : ''}...`);
+ let pages = 0, updated = 0, withImg = 0, multiImg = 0, errs = 0;
+ let idx = 0;
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const r = rows[i];
+ try {
+ const html = await httpGet(r.url);
+ const { all_images, room_setting_url } = extractGrahamImages(html, r.mfr_sku, r.url);
+ pages++;
+ if (all_images.length) withImg++;
+ if (all_images.length > 1) multiImg++;
+ if (DRY_RUN) {
+ console.log(` ${r.dw_sku} (${r.mfr_sku}) -> ${all_images.length} img${room_setting_url ? ' +room' : ''}`);
+ all_images.forEach((u) => console.log(` ${u}`));
+ } else if (all_images.length) {
+ // additive: only set all_images where it is still empty (guards concurrent fills)
+ const res = await pool.query(
+ `UPDATE ${TABLE}
+ SET all_images = $1,
+ room_setting_url = COALESCE(NULLIF(room_setting_url,''), $2),
+ updated_at = now()
+ WHERE id = $3 AND (all_images IS NULL OR all_images = '' OR all_images = '[]')`,
+ [JSON.stringify(all_images), room_setting_url, r.id]
+ );
+ if (res.rowCount) updated++;
+ }
+ if (pages % 25 === 0) console.log(` ${pages}/${rows.length} pages | ${updated} updated | ${withImg} w/img | ${multiImg} multi | ${errs} errs`);
+ await sleep(350);
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0) console.error(` err at ${r.url}: ${e.message}`);
+ await sleep(800);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: CONCURRENCY }, worker));
+ console.log(`\nDONE. pages=${pages} updated=${updated} with_img=${withImg} multi_img=${multiImg} errors=${errs}`);
+ if (pool) await pool.end();
+}
+
+function DATABASE_URL_PRESENT() { return !!process.env.DATABASE_URL; }
+
+main().catch((e) => { console.error('FATAL', e); process.exit(1); });
diff --git a/reference/recrawl-osborne.js b/reference/recrawl-osborne.js
new file mode 100644
index 0000000..8623a61
--- /dev/null
+++ b/reference/recrawl-osborne.js
@@ -0,0 +1,77 @@
+#!/usr/bin/env node
+// Re-crawl driver: pulls every distinct osborne product_url from the DB and
+// re-runs the FIXED extraction (images + specs) via the deployed scraper's
+// exported functions, then upserts into osborne_catalog. Avoids the flaky
+// browser collection-listing crawl by reusing known product URLs.
+require('dotenv').config({ path: '/root/DW-Agents/dw-price-stock/.env' });
+const fs = require('fs');
+const https = require('https');
+const { upsertProduct, pool } = require('/root/DW-Agents/vendor-scrapers/scraper-utils');
+
+// Load extractFromHtml from the deployed scraper without running its main().
+// Write the temp module INSIDE vendor-scrapers so node_modules (dotenv/pg) resolve.
+const EXTRACT_TMP = '/root/DW-Agents/vendor-scrapers/_osb_extract.js';
+let src = fs.readFileSync('/root/DW-Agents/vendor-scrapers/scrape-osborne.js', 'utf8')
+ .replace(/main\(\)\.catch\([\s\S]*$/, 'module.exports={extractFromHtml};\n');
+fs.writeFileSync(EXTRACT_TMP, src);
+const { extractFromHtml } = require(EXTRACT_TMP);
+
+const SITE_BASE = 'https://www.osborneandlittle.com';
+const TABLE = 'osborne_catalog';
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
+
+function httpGet(url, maxRedirects = 4) {
+ return new Promise((resolve, reject) => {
+ const req = https.get(url, { timeout: 25000, headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'en-US,en;q=0.9' } }, (res) => {
+ if ((res.statusCode === 301 || res.statusCode === 302) && res.headers.location && maxRedirects > 0) {
+ const loc = res.headers.location.startsWith('http') ? res.headers.location : SITE_BASE + res.headers.location;
+ res.resume();
+ return httpGet(loc, maxRedirects - 1).then(resolve).catch(reject);
+ }
+ if (res.statusCode !== 200) { res.resume(); return reject(new Error('HTTP ' + res.statusCode)); }
+ let d = ''; res.on('data', c => d += c); res.on('end', () => resolve(d)); res.on('error', reject);
+ });
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ });
+}
+const sleep = (ms) => new Promise(r => setTimeout(r, ms + Math.floor(Math.random() * 250)));
+
+(async () => {
+ const { rows } = await pool.query(
+ `SELECT DISTINCT split_part(product_url,'?',1) AS url FROM ${TABLE} WHERE product_url IS NOT NULL AND product_url <> '' ORDER BY 1`
+ );
+ console.log(`Re-crawling ${rows.length} distinct osborne product pages...`);
+ let pages = 0, skus = 0, withImg = 0, withSpecs = 0, errs = 0;
+ const concurrency = 3;
+ let idx = 0;
+ async function worker() {
+ while (idx < rows.length) {
+ const i = idx++;
+ const url = rows[i].url;
+ try {
+ const html = await httpGet(url);
+ const products = extractFromHtml(html, url);
+ for (const p of products) {
+ if (!p.mfr_sku) continue;
+ const id = await upsertProduct(TABLE, p);
+ if (id) {
+ skus++;
+ if (p.image_url) withImg++;
+ if (p.specs) withSpecs++;
+ }
+ }
+ pages++;
+ if (pages % 25 === 0) console.log(` ${pages}/${rows.length} pages | ${skus} skus | ${withImg} w/img | ${withSpecs} w/specs | ${errs} errs`);
+ await sleep(350);
+ } catch (e) {
+ errs++;
+ if (errs % 10 === 0) console.error(` err at ${url}: ${e.message}`);
+ await sleep(800);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: concurrency }, worker));
+ console.log(`\nDONE. pages=${pages} skus_upserted=${skus} with_img=${withImg} with_specs=${withSpecs} errors=${errs}`);
+ await pool.end();
+})().catch(e => { console.error('FATAL', e); pool.end().catch(() => {}); process.exit(1); });
diff --git a/reference/scrape-cowtan.js b/reference/scrape-cowtan.js
new file mode 100644
index 0000000..2e119f1
--- /dev/null
+++ b/reference/scrape-cowtan.js
@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+// Cowtan & Tout Catalog Scraper - designs.cowtan.com AJAX API
+// Target table: cowtan_tout_catalog
+const https = require('https');
+const { pool, upsertProduct, wait, closePool, getCatalogCount } = require('./scraper-utils');
+const TABLE = 'cowtan_tout_catalog';
+const API_BASE = 'https://designs.cowtan.com';
+const IMG_BASE = 'https://d2mq91o692rj7w.cloudfront.net/images/Products';
+const BRAND_MAP = { 1: 'Cowtan & Tout', 2: 'Colefax and Fowler', 3: 'Jane Churchill', 5: 'Larsen', 6: 'Manuel Canovas' };
+const TYPE_MAP = { W: 'Wallcovering', F: 'Fabric', T: 'Trimming' };
+let totalInserted = 0, totalErrors = 0;
+
+function httpPost(url, body) {
+ return new Promise(function(resolve, reject) {
+ var parsed = new URL(url);
+ var opts = { hostname: parsed.hostname, port: 443, path: parsed.pathname,
+ method: 'POST', timeout: 30000,
+ headers: { 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': Buffer.byteLength(body),
+ 'User-Agent': 'Mozilla/5.0', 'Accept': 'application/json', 'X-Requested-With': 'XMLHttpRequest' } };
+ var req = https.request(opts, function(res) {
+ if (res.statusCode === 301 || res.statusCode === 302) {
+ return httpPost(res.headers.location, body).then(resolve).catch(reject);
+ }
+ var data = '';
+ res.on('data', function(c) { data += c; });
+ res.on('end', function() { try { resolve(JSON.parse(data)); } catch(e) { reject(new Error('JSON: ' + e.message + ' (len=' + data.length + ')')); } });
+ res.on('error', reject);
+ });
+ req.on('error', reject); req.on('timeout', function() { req.destroy(); reject(new Error('timeout')); });
+ req.write(body); req.end();
+ });
+}
+
+function buildBody(pi, tv) { var p = ['Keywords=','NewOnly=false']; ['F','W','T'].forEach(function(t,i) {
+ p.push('Types%5B'+i+'%5D.Value='+t); p.push('Types%5B'+i+'%5D.Selected='+(t===tv?'true':'false')); });
+ p.push('PageIndex='+pi); return p.join('&'); }
+
+async function scrapeType(tv) { console.log('\n Fetching: '+TYPE_MAP[tv]+'...'); var all = [];
+ var f = await httpPost(API_BASE+'/Search', buildBody(0, tv)); var tot = f.TotalResults||0;
+ console.log(' Total: '+tot); if (!f.Products||!f.Products.length) return all;
+ var pp = f.Products[0]||[]; all.push.apply(all, pp); console.log(' Page 0: '+pp.length);
+ var mx = Math.min(Math.ceil(tot/100), 200);
+ for (var pi=1; pi<mx; pi++) { try { await wait(200,150);
+ var pg = await httpPost(API_BASE+'/Search', buildBody(pi, tv));
+ if (!pg.Products||!pg.Products.length) break; var it = pg.Products[0]||[]; if (!it.length) break;
+ all.push.apply(all, it); if ((pi+1)%10===0) console.log(' Page '+pi+': '+all.length);
+ } catch(e) { console.error(' Pg '+pi+': '+e.message); totalErrors++; await wait(2000); } }
+ console.log(' '+TYPE_MAP[tv]+' total: '+all.length); return all; }
+
+function mapProduct(r, tv) { if (!r.StockCode) return null;
+ return { mfr_sku: r.StockCode, pattern_name: r.ProductName||null, color_name: r.ColourName||null,
+ collection: r.PatternBook||null, material: BRAND_MAP[r.BrandCode]||'Cowtan & Tout',
+ product_type: TYPE_MAP[tv], price_retail: r.SamplePrice||null,
+ image_url: IMG_BASE+'/'+r.StockCode.replace(/\//g,'-')+'.jpg',
+ product_url: 'https://designs.cowtan.com/Design/'+r.StockCode.replace(/\//g,'-'), in_stock: true }; }
+
+async function main() { console.log('='.repeat(70)); console.log(' COWTAN & TOUT CATALOG SCRAPER'); console.log('='.repeat(70));
+ var t0 = Date.now();
+ await pool.query('CREATE TABLE IF NOT EXISTS '+TABLE+' (id SERIAL PRIMARY KEY, mfr_sku TEXT UNIQUE, dw_sku TEXT, pattern_name TEXT, color_name TEXT, collection TEXT, product_type TEXT, width TEXT, length TEXT, repeat_v TEXT, repeat_h TEXT, material TEXT, price_retail NUMERIC(10,2), price_trade NUMERIC(10,2), image_url TEXT, product_url TEXT, in_stock BOOLEAN DEFAULT TRUE, discontinued BOOLEAN DEFAULT FALSE, shopify_product_id TEXT, last_scraped TIMESTAMPTZ, created_at TIMESTAMPTZ DEFAULT NOW(), updated_at TIMESTAMPTZ DEFAULT NOW())');
+ console.log(' Table ready.'); var wp = await scrapeType('W'); await wait(1000,500); var fp = await scrapeType('F');
+ console.log('\n Upserting...'); var ar = [];
+ wp.forEach(function(r){var p=mapProduct(r,'W');if(p)ar.push(p);});
+ fp.forEach(function(r){var p=mapProduct(r,'F');if(p)ar.push(p);});
+ var s={},u=[]; ar.forEach(function(p){if(!s[p.mfr_sku]){s[p.mfr_sku]=true;u.push(p);}});
+ console.log(' Unique: '+u.length);
+ for(var i=0;i<u.length;i++){try{var id=await upsertProduct(TABLE,u[i]);if(id)totalInserted++;}catch(e){totalErrors++;}
+ if((i+1)%500===0)console.log(' DB: '+(i+1)+'/'+u.length+' ins='+totalInserted);}
+ var el=((Date.now()-t0)/1000).toFixed(1); var ct=await getCatalogCount(TABLE);
+ console.log('\n'+'='.repeat(70)); console.log(' DONE: wall='+wp.length+' fab='+fp.length+' ins='+totalInserted+' err='+totalErrors+' db='+ct);
+ console.log(' Time: '+el+'s'); console.log('='.repeat(70)); await closePool(); }
+main().catch(function(e){console.error('FATAL:',e);closePool().then(function(){process.exit(1);});});
diff --git a/reference/scrape-hollywood.js b/reference/scrape-hollywood.js
new file mode 100644
index 0000000..50d2b60
--- /dev/null
+++ b/reference/scrape-hollywood.js
@@ -0,0 +1,346 @@
+#!/usr/bin/env node
+// ============================================================================
+// Hollywood Wallcoverings Scraper
+// ============================================================================
+// Strategy: Shopify JSON API from designerwallcoverings.com
+// - hollywoodwallcoverings.com redirects to designerwallcoverings.com
+// - Single collection: hollywood-wallcovering-collection
+// - ~5000+ products, paginate products.json?limit=250&page=N
+// - Extract: SKU, pattern, color, collection, material, price, image, URL
+//
+// Target table: hollywood_catalog
+// ============================================================================
+
+const { pool, upsertProduct, wait, closePool } = require('./scraper-utils');
+const https = require('https');
+
+const TABLE = 'hollywood_catalog';
+const SHOP_BASE = 'https://www.designerwallcoverings.com';
+const COLLECTION_SLUG = 'hollywood-wallcovering-collection';
+
+let totalInserted = 0;
+let totalErrors = 0;
+let totalSkipped = 0;
+
+// ============================================================================
+// HTTP helper
+// ============================================================================
+
+const httpsAgent = new https.Agent({ rejectUnauthorized: false });
+
+function httpGetJson(url) {
+ return new Promise((resolve, reject) => {
+ const urlObj = new URL(url);
+ const options = {
+ hostname: urlObj.hostname,
+ path: urlObj.pathname + urlObj.search,
+ method: 'GET',
+ agent: httpsAgent,
+ headers: {
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
+ 'Accept': 'application/json',
+ },
+ timeout: 30000,
+ };
+
+ const req = https.request(options, (res) => {
+ if (res.statusCode === 301 || res.statusCode === 302) {
+ return httpGetJson(res.headers.location).then(resolve).catch(reject);
+ }
+ let data = '';
+ res.on('data', chunk => data += chunk);
+ res.on('end', () => {
+ try {
+ resolve(JSON.parse(data));
+ } catch (e) {
+ reject(new Error(`JSON parse error at ${url}: ${e.message}`));
+ }
+ });
+ });
+
+ req.on('error', reject);
+ req.on('timeout', () => { req.destroy(); reject(new Error('timeout')); });
+ req.end();
+ });
+}
+
+// ============================================================================
+// Parse a Shopify product into catalog format
+// ============================================================================
+
+function parseShopifyProduct(product) {
+ const title = product.title || '';
+ const tags = product.tags || [];
+ const handle = product.handle || '';
+
+ // Title format: "Abbingdon Type II Vinyl Wallpaper | Hollywood Wallcoverings"
+ // or: "Pattern Name - Color | Hollywood Wallcoverings"
+ let pattern_name = null;
+ let color_name = null;
+
+ // Remove vendor suffix
+ let cleanTitle = title.replace(/\s*\|\s*Hollywood\s+Wallcoverings\s*/i, '').trim();
+
+ // Try "Pattern - Color" format
+ const dashMatch = cleanTitle.match(/^(.+?)\s*-\s*(.+?)(?:\s+(?:Type\s+II|Vinyl|Wallpaper|Wallcovering))?$/i);
+ if (dashMatch) {
+ pattern_name = dashMatch[1].trim();
+ color_name = dashMatch[2].trim();
+ } else {
+ // Use full title as pattern name, clean up type suffixes
+ pattern_name = cleanTitle
+ .replace(/\s+Type\s+II\s+Vinyl\s+Wallpaper/i, '')
+ .replace(/\s+Type\s+II\s+Vinyl\s+Wallcovering/i, '')
+ .replace(/\s+Wallpaper/i, '')
+ .replace(/\s+Wallcovering/i, '')
+ .trim();
+ }
+
+ // Try to extract color from tags
+ if (!color_name) {
+ const colorTags = tags.filter(t =>
+ !t.match(/Hollywood|Wallcovering|Wallpaper|vinyl|display_variant|Type\s*II/i)
+ && !t.match(/^\d/)
+ && t.length > 2
+ && t.length < 30
+ );
+ // Use the tag matching the pattern name to find the color
+ const patternTag = tags.find(t => t === pattern_name);
+ if (patternTag) {
+ // Pattern tag found, color is the next non-matching descriptive tag
+ const colorCandidates = colorTags.filter(t => t !== pattern_name);
+ if (colorCandidates.length > 0) color_name = colorCandidates[0];
+ }
+ }
+
+ // Variants - get SKU and price
+ const variants = product.variants || [];
+ let mfr_sku = null;
+ let price_retail = null;
+
+ // Prefer non-sample variant
+ const fullVariant = variants.find(v => !v.sku?.toLowerCase().includes('sample'));
+ const sampleVariant = variants.find(v => v.sku?.toLowerCase().includes('sample'));
+
+ if (fullVariant) {
+ mfr_sku = fullVariant.sku;
+ price_retail = parseFloat(fullVariant.price) || null;
+ } else if (sampleVariant) {
+ mfr_sku = sampleVariant.sku.replace(/-[Ss]ample$/i, '');
+ price_retail = null;
+ }
+
+ if (!mfr_sku) mfr_sku = handle;
+
+ // Extract material from tags
+ let material = null;
+ if (tags.some(t => t.match(/vinyl/i))) material = 'Vinyl';
+ else if (tags.some(t => t.match(/100%\s*vinyl/i))) material = '100% Vinyl';
+ else if (tags.some(t => t.match(/Type\s*II/i))) material = 'Type II Vinyl';
+
+ // Product type
+ const product_type = product.product_type || 'Commercial Wallcoverings';
+
+ // Collection detection from tags
+ let collection = 'Hollywood Wallcoverings';
+ // Some tags might indicate sub-collections
+ const collTag = tags.find(t => t.match(/collection/i) && !t.match(/Hollywood/i));
+ if (collTag) collection = collTag;
+
+ // Images — capture the FULL array, not just the hero.
+ // The Shopify products.json feed exposes product.images[] (often 8+ per SKU);
+ // the old code kept only images[0] and discarded the rest. We now persist all
+ // full-resolution srcs in all_images (JSON array string) and keep image_url as
+ // the first/hero for backward compatibility.
+ const imageSrcs = (product.images || [])
+ .map(i => i && i.src)
+ .filter(Boolean);
+ const image_url = imageSrcs[0] || null;
+ const all_images = imageSrcs.length ? JSON.stringify(imageSrcs) : null;
+
+ // Product URL
+ const product_url = `${SHOP_BASE}/products/${handle}`;
+
+ // ALL DATA — capture additional feed fields the old parser dropped, into a
+ // specs jsonb blob so nothing useful from the source is thrown away.
+ const allVariantSkus = variants
+ .map(v => v && v.sku)
+ .filter(Boolean);
+ const specs = {
+ shopify_product_id: product.id ? String(product.id) : null,
+ handle,
+ title,
+ body_html: product.body_html || null, // spec/description text
+ vendor: product.vendor || null,
+ product_type: product.product_type || null,
+ published_at: product.published_at || null,
+ created_at: product.created_at || null,
+ updated_at: product.updated_at || null,
+ tags,
+ options: product.options || null, // e.g. Title / sample-vs-roll
+ variant_skus: allVariantSkus, // all variant SKUs (roll + sample)
+ image_count: imageSrcs.length,
+ image_srcs: imageSrcs, // redundant w/ all_images, kept in specs for completeness
+ };
+
+ return {
+ mfr_sku,
+ pattern_name,
+ color_name,
+ collection,
+ product_type,
+ material,
+ price_retail,
+ image_url,
+ all_images,
+ product_url,
+ shopify_product_id: product.id ? String(product.id) : null,
+ specs: JSON.stringify(specs),
+ };
+}
+
+// ============================================================================
+// MAIN
+// ============================================================================
+
+async function main() {
+ console.log('='.repeat(70));
+ console.log(' HOLLYWOOD WALLCOVERINGS SCRAPER');
+ console.log('='.repeat(70));
+ console.log(` Source: Shopify JSON API (${SHOP_BASE})`);
+ console.log(` Collection: ${COLLECTION_SLUG}`);
+ console.log(` Target table: ${TABLE}`);
+ console.log('='.repeat(70));
+
+ const startTime = Date.now();
+
+ // Ensure table exists
+ try {
+ await pool.query(`SELECT 1 FROM ${TABLE} LIMIT 1`);
+ console.log(`\n Table "${TABLE}" exists.`);
+ } catch (err) {
+ console.log(`\n Table "${TABLE}" does not exist, creating...`);
+ await pool.query(`
+ CREATE TABLE IF NOT EXISTS ${TABLE} (
+ id SERIAL PRIMARY KEY,
+ mfr_sku TEXT UNIQUE,
+ dw_sku TEXT,
+ pattern_name TEXT,
+ color_name TEXT,
+ collection TEXT,
+ product_type TEXT,
+ width TEXT,
+ length TEXT,
+ repeat_v TEXT,
+ repeat_h TEXT,
+ material TEXT,
+ price_retail NUMERIC(10,2),
+ price_trade NUMERIC(10,2),
+ image_url TEXT,
+ product_url TEXT,
+ in_stock BOOLEAN DEFAULT TRUE,
+ discontinued BOOLEAN DEFAULT FALSE,
+ shopify_product_id TEXT,
+ last_scraped TIMESTAMPTZ,
+ created_at TIMESTAMPTZ DEFAULT NOW(),
+ updated_at TIMESTAMPTZ DEFAULT NOW()
+ );
+ `);
+ console.log(` Table created.`);
+ }
+
+ // Ensure the columns we now write exist (idempotent — safe on every run).
+ // all_images stores the full image array (JSON), specs stores the rest of the
+ // feed (body_html, all variant SKUs, options, etc.) so no source data is lost.
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS all_images TEXT`);
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS shopify_product_id TEXT`);
+ await pool.query(`ALTER TABLE ${TABLE} ADD COLUMN IF NOT EXISTS specs JSONB`);
+
+ // Paginate through collection
+ let page = 1;
+ let grandTotal = 0;
+ const maxPages = 60; // Safety limit (collection ~3,300 products / 250 = ~14 pages)
+
+ while (page <= maxPages) {
+ try {
+ const url = `${SHOP_BASE}/collections/${COLLECTION_SLUG}/products.json?limit=250&page=${page}`;
+ const data = await httpGetJson(url);
+ const products = data.products || [];
+
+ if (products.length === 0) break;
+
+ for (const product of products) {
+ const parsed = parseShopifyProduct(product);
+ if (!parsed || !parsed.mfr_sku) {
+ totalSkipped++;
+ continue;
+ }
+
+ try {
+ const id = await upsertProduct(TABLE, parsed);
+ if (id) totalInserted++;
+ else totalSkipped++;
+ } catch (err) {
+ totalErrors++;
+ }
+ grandTotal++;
+ }
+
+ console.log(` Page ${page}: ${products.length} products (total: ${grandTotal}, inserted: ${totalInserted})`);
+
+ if (products.length < 250) break;
+ page++;
+ await wait(500, 300);
+ } catch (err) {
+ console.error(` Error on page ${page}: ${err.message}`);
+ totalErrors++;
+ // Retry once after delay
+ await wait(3000, 1000);
+ try {
+ const url = `${SHOP_BASE}/collections/${COLLECTION_SLUG}/products.json?limit=250&page=${page}`;
+ const data = await httpGetJson(url);
+ const products = data.products || [];
+ if (products.length === 0) break;
+ for (const product of products) {
+ const parsed = parseShopifyProduct(product);
+ if (parsed && parsed.mfr_sku) {
+ const id = await upsertProduct(TABLE, parsed);
+ if (id) totalInserted++;
+ grandTotal++;
+ }
+ }
+ console.log(` Page ${page} (retry): ${products.length} products`);
+ } catch (retryErr) {
+ console.error(` Retry failed for page ${page}, skipping.`);
+ }
+ page++;
+ await wait(1000, 500);
+ }
+ }
+
+ // Final stats
+ const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
+ let count = 0;
+ try {
+ const res = await pool.query(`SELECT COUNT(*) as cnt FROM ${TABLE}`);
+ count = parseInt(res.rows[0].cnt);
+ } catch (_) {}
+
+ console.log('\n' + '='.repeat(70));
+ console.log(' SCRAPE COMPLETE - HOLLYWOOD WALLCOVERINGS');
+ console.log('='.repeat(70));
+ console.log(` Products processed: ${grandTotal}`);
+ console.log(` Products inserted/updated: ${totalInserted}`);
+ console.log(` Skipped: ${totalSkipped}`);
+ console.log(` Errors: ${totalErrors}`);
+ console.log(` Total in ${TABLE}: ${count}`);
+ console.log(` Duration: ${elapsed}s`);
+ console.log('='.repeat(70));
+
+ await closePool();
+}
+
+main().catch(err => {
+ console.error('FATAL:', err);
+ closePool().then(() => process.exit(1));
+});
(oldest)
·
back to Dw Cowtan Recrawl
·
recrawl-cowtan: self-heal worklist WHERE re-selects pinteres 64d92ba →